23836075406-2909

Committed 01 Apr 2026 06:51AM UTC coverage: 69.084% (+0.007%) from 69.077%

Build # 23836075406-2909

Build Type

push

github

Committed by

tstack

Commit Message

[tags] only save user-provided tags in the session, not format-provided

Coverage Stats

60 of 76 new or added lines in 14 files covered. (78.95%)

2 existing lines in 2 files now uncovered.

53287 of 77134 relevant lines covered (69.08%)

535481.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.91

/src/logfile_sub_source.cc

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <algorithm>
#include <chrono>
#include <functional>
#include <future>
#include <optional>
#include <string>
#include <unordered_set>
#include <vector>

#include "logfile_sub_source.hh"

#include <sqlite3.h>

#include "base/ansi_scrubber.hh"
#include "base/ansi_vars.hh"
#include "base/distributed_slice.hh"
#include "base/injector.hh"
#include "base/itertools.hh"
#include "base/string_util.hh"
#include "bookmarks.hh"
#include "bookmarks.json.hh"
#include "command_executor.hh"
#include "config.h"
#include "field_overlay_source.hh"
#include "file_collection.hh"
#include "hasher.hh"
#include "k_merge_tree.h"
#include "lnav_util.hh"
#include "log_accel.hh"
#include "logfile_sub_source.cfg.hh"
#include "logline_window.hh"
#include "md2attr_line.hh"
#include "ptimec.hh"
#include "scn/scan.h"
#include "shlex.hh"
#include "sql_util.hh"
#include "sqlitepp.client.hh"
#include "sqlitepp.hh"
#include "tlx/container/btree_set.hpp"
#include "vtab_module.hh"
#include "yajlpp/yajlpp.hh"
#include "yajlpp/yajlpp_def.hh"

using namespace std::chrono_literals;
using namespace std::string_literals;
using namespace lnav::roles::literals;

const DIST_SLICE(bm_types) bookmark_type_t logfile_sub_source::BM_FILES("file");

static int
pretty_sql_callback(exec_context& ec, sqlite3_stmt* stmt)
{
    if (!sqlite3_stmt_busy(stmt)) {
        return 0;
    }

    const auto ncols = sqlite3_column_count(stmt);

    for (int lpc = 0; lpc < ncols; lpc++) {
        if (!ec.ec_accumulator->empty()) {
            ec.ec_accumulator->append(", ");
        }

        const char* res = (const char*) sqlite3_column_text(stmt, lpc);
        if (res == nullptr) {
            continue;
        }

        ec.ec_accumulator->append(res);
    }

    for (int lpc = 0; lpc < ncols; lpc++) {
        const auto* colname = sqlite3_column_name(stmt, lpc);
        auto* raw_value = sqlite3_column_value(stmt, lpc);
        auto value_type = sqlite3_value_type(raw_value);
        scoped_value_t value;

        switch (value_type) {
            case SQLITE_INTEGER:
                value = (int64_t) sqlite3_value_int64(raw_value);
                break;
            case SQLITE_FLOAT:
                value = sqlite3_value_double(raw_value);
                break;
            case SQLITE_NULL:
                value = null_value_t{};
                break;
            default:
                value = string_fragment::from_bytes(
                    sqlite3_value_text(raw_value),
                    sqlite3_value_bytes(raw_value));
                break;
        }
        if (!ec.ec_local_vars.empty() && !ec.ec_dry_run) {
            if (sql_ident_needs_quote(colname)) {
                continue;
            }
            auto& vars = ec.ec_local_vars.top();

            if (vars.find(colname) != vars.end()) {
                continue;
            }

            if (value.is<string_fragment>()) {
                value = value.get<string_fragment>().to_string();
            }
            vars[colname] = value;
        }
    }
    return 0;
}

static std::future<std::string>
pretty_pipe_callback(exec_context& ec, const std::string& cmdline, auto_fd& fd)
{
    auto retval = std::async(std::launch::async, [&]() {
        char buffer[1024];
        std::ostringstream ss;
        ssize_t rc;

        while ((rc = read(fd, buffer, sizeof(buffer))) > 0) {
            ss.write(buffer, rc);
        }

        auto retval = ss.str();

        if (endswith(retval, "\n")) {
            retval.resize(retval.length() - 1);
        }

        return retval;
    });

    return retval;
}

logfile_sub_source::logfile_sub_source()
    : text_sub_source(1), lnav_config_listener(__FILE__),
      lss_meta_grepper(*this), lss_location_history(*this)
{
    this->tss_supports_filtering = true;
    this->clear_line_size_cache();
    this->clear_min_max_row_times();
}

std::shared_ptr<logfile>
logfile_sub_source::find(const char* fn, content_line_t& line_base)
{
    std::shared_ptr<logfile> retval = nullptr;

    line_base = content_line_t(0);
    for (auto iter = this->lss_files.begin();
         iter != this->lss_files.end() && retval == nullptr;
         ++iter)
    {
        auto& ld = *(*iter);
        auto* lf = ld.get_file_ptr();

        if (lf == nullptr) {
            continue;
        }
        if (strcmp(lf->get_filename_as_string().c_str(), fn) == 0) {
            retval = ld.get_file();
        } else {
            line_base += content_line_t(MAX_LINES_PER_FILE);
        }
    }

    return retval;
}

struct filtered_logline_cmp {
    filtered_logline_cmp(const logfile_sub_source& lc) : llss_controller(lc) {}

    bool operator()(const uint32_t& lhs, const uint32_t& rhs) const
    {
        auto cl_lhs = llss_controller.lss_index[lhs].value();
        auto cl_rhs = llss_controller.lss_index[rhs].value();
        const auto* ll_lhs = this->llss_controller.find_line(cl_lhs);
        const auto* ll_rhs = this->llss_controller.find_line(cl_rhs);

        if (ll_lhs == nullptr) {
            return true;
        }
        if (ll_rhs == nullptr) {
            return false;
        }
        return (*ll_lhs) < (*ll_rhs);
    }

    bool operator()(const uint32_t& lhs, const timeval& rhs) const
    {
        const auto cl_lhs = llss_controller.lss_index[lhs].value();
        const auto* ll_lhs = this->llss_controller.find_line(cl_lhs);

        if (ll_lhs == nullptr) {
            return true;
        }
        return (*ll_lhs) < rhs;
    }

    const logfile_sub_source& llss_controller;
};

std::optional<vis_line_t>
logfile_sub_source::find_from_time(const timeval& start) const
{
    const auto lb = std::lower_bound(this->lss_filtered_index.begin(),
                                     this->lss_filtered_index.end(),
                                     start,
                                     filtered_logline_cmp(*this));
    if (lb != this->lss_filtered_index.end()) {
        auto retval = std::distance(this->lss_filtered_index.begin(), lb);
        return vis_line_t(retval);
    }

    return std::nullopt;
}

line_info
logfile_sub_source::text_value_for_line(textview_curses& tc,
                                        int row,
                                        std::string& value_out,
                                        line_flags_t flags)
{
    if (this->lss_indexing_in_progress) {
        value_out = "";
        this->lss_token_al.al_attrs.clear();
        return {};
    }

    line_info retval;
    content_line_t line(0);

    require_ge(row, 0);
    require_lt((size_t) row, this->lss_filtered_index.size());

    line = this->at(vis_line_t(row));

    if (flags & RF_RAW) {
        auto lf = this->find(line);
        auto ll = lf->begin() + line;
        retval.li_file_range = lf->get_file_range(ll, false);
        retval.li_level = ll->get_msg_level();
        // retval.li_timestamp = ll->get_timeval();
        retval.li_partial = false;
        retval.li_utf8_scan_result.usr_has_ansi = ll->has_ansi();
        retval.li_utf8_scan_result.usr_message = ll->is_valid_utf() ? nullptr
                                                                    : "bad";
        // timeval start_time, end_time;
        // gettimeofday(&start_time, NULL);
        value_out = lf->read_line(lf->begin() + line)
                        .map([](auto sbr) { return to_string(sbr); })
                        .unwrapOr({});
        // gettimeofday(&end_time, NULL);
        // timeval diff = end_time - start_time;
        // log_debug("read time %d.%06d %s:%d", diff.tv_sec, diff.tv_usec,
        // lf->get_filename().c_str(), line);
        return retval;
    }

    require_false(this->lss_in_value_for_line);

    this->lss_in_value_for_line = true;
    this->lss_token_flags = flags;
    this->lss_token_file_data = this->find_data(line);
    this->lss_token_file = (*this->lss_token_file_data)->get_file();
    this->lss_token_line = this->lss_token_file->begin() + line;

    this->lss_token_al.clear();
    this->lss_token_values.clear();
    this->lss_share_manager.invalidate_refs();
    if (flags & RF_FULL) {
        shared_buffer_ref sbr;

        this->lss_token_file->read_full_message(this->lss_token_line, sbr);
        this->lss_token_al.al_string = to_string(sbr);
        if (sbr.get_metadata().m_has_ansi) {
            scrub_ansi_string(this->lss_token_al.al_string,
                              &this->lss_token_al.al_attrs);
            sbr.get_metadata().m_has_ansi = false;
        }
    } else {
        auto sub_opts = subline_options{};
        sub_opts.scrub_invalid_utf8 = false;
        this->lss_token_al.al_string
            = this->lss_token_file->read_line(this->lss_token_line, sub_opts)
                  .map([](auto sbr) { return to_string(sbr); })
                  .unwrapOr({});
        if (this->lss_token_line->has_ansi()) {
            scrub_ansi_string(this->lss_token_al.al_string,
                              &this->lss_token_al.al_attrs);
        }
    }
    this->lss_token_shifts.clear();

    auto format = this->lss_token_file->get_format_ptr();

    auto& sbr = this->lss_token_values.lvv_sbr;

    sbr.share(this->lss_share_manager,
              (char*) this->lss_token_al.al_string.c_str(),
              this->lss_token_al.al_string.size());
    format->annotate(this->lss_token_file.get(),
                     line,
                     this->lss_token_al.al_attrs,
                     this->lss_token_values);

    auto src_file_attr
        = find_string_attr(this->lss_token_al.al_attrs, &SA_SRC_FILE);
    if (src_file_attr != this->lss_token_al.al_attrs.end()) {
        auto lr = src_file_attr->sa_range;
        lr.lr_end = lr.lr_start + 1;
        auto break_ta = text_attrs::with_underline();
        this->lss_token_al.with_attr({lr, VC_STYLE.value(break_ta)})
            .with_attr({lr,
                        VC_COMMAND.value(ui_command{
                            source_location{},
                            "|lnav-src-loc-handler $mouse_button",
                        })});
        if (!this->lss_breakpoints.empty()) {
            if (this->lss_token_values.lvv_src_line_value) {
                auto h = hasher();
                h.update(format->get_name().to_string_fragment());
                h.update(this->lss_token_values.lvv_src_file_value.value());
                h.update(this->lss_token_values.lvv_src_line_value.value());

                auto schema = h.to_string();
                auto& breakpoints = this->lss_breakpoints;
                auto it = breakpoints.find(schema);
                if (it != breakpoints.end()) {
                    lr.lr_end = lr.lr_start;
                    this->lss_token_al.insert(
                        lr.lr_start,
                        it->second.bp_enabled ? ui_icon_t::breakpoint
                                              : ui_icon_t::disabled_breakpoint);
                    lr.lr_end += 2;
                    this->lss_token_al.with_attr(
                        {lr,
                         VC_COMMAND.value(ui_command{
                             source_location{},
                             "|lnav-breakpoint-handler $mouse_button",
                         })});
                    this->lss_token_values.shift_origins_by(lr, 2);
                }
            }
        }
    }

    value_out = this->lss_token_al.al_string;

    for (const auto& hl : format->lf_highlighters) {
        auto hl_range = line_range{0, -1};
        auto value_iter = this->lss_token_values.lvv_values.end();
        if (!hl.h_field.empty()) {
            value_iter = std::find_if(this->lss_token_values.lvv_values.begin(),
                                      this->lss_token_values.lvv_values.end(),
                                      logline_value_name_cmp(&hl.h_field));
            if (value_iter == this->lss_token_values.lvv_values.end()) {
                continue;
            }
            hl_range = value_iter->lv_origin;
        }
        if (hl.annotate(this->lss_token_al, hl_range)
            && value_iter != this->lss_token_values.lvv_values.end())
        {
            value_iter->lv_highlighted = true;
        }
    }

    for (const auto& hl : this->lss_highlighters) {
        auto hl_range = line_range{0, -1};
        auto value_iter = this->lss_token_values.lvv_values.end();
        if (!hl.h_field.empty()) {
            value_iter = std::find_if(this->lss_token_values.lvv_values.begin(),
                                      this->lss_token_values.lvv_values.end(),
                                      logline_value_name_cmp(&hl.h_field));
            if (value_iter == this->lss_token_values.lvv_values.end()) {
                continue;
            }
            hl_range = value_iter->lv_origin;
        }
        if (hl.annotate(this->lss_token_al, hl_range)
            && value_iter != this->lss_token_values.lvv_values.end())
        {
            value_iter->lv_highlighted = true;
        }
    }

    if (flags & RF_REWRITE) {
        exec_context ec(
            &this->lss_token_values, pretty_sql_callback, pretty_pipe_callback);
        std::string rewritten_line;
        db_label_source rewrite_label_source;

        ec.with_perms(exec_context::perm_t::READ_ONLY);
        ec.ec_local_vars.push(std::map<std::string, scoped_value_t>());
        ec.ec_top_line = vis_line_t(row);
        ec.ec_label_source_stack.push_back(&rewrite_label_source);
        add_ansi_vars(ec.ec_global_vars);
        add_global_vars(ec);
        format->rewrite(ec, sbr, this->lss_token_al.al_attrs, rewritten_line);
        this->lss_token_al.al_string.assign(rewritten_line);
        value_out = this->lss_token_al.al_string;
    }

    {
        auto lr = line_range{0, (int) this->lss_token_al.al_string.length()};
        this->lss_token_al.al_attrs.emplace_back(lr, SA_ORIGINAL_LINE.value());
    }

    // Replace VALUE_TIMESTAMP fields right-to-left so that origins
    // for earlier fields remain valid.
    if (format->lf_date_time.dts_fmt_lock != -1) {
        for (auto lv_iter = this->lss_token_values.lvv_values.rbegin();
             lv_iter != this->lss_token_values.lvv_values.rend();
             ++lv_iter)
        {
            if (lv_iter->lv_meta.lvm_kind != value_kind_t::VALUE_TIMESTAMP
                || !lv_iter->lv_origin.is_valid())
            {
                continue;
            }

            auto ts_str = lv_iter->to_string();
            value_out.replace(lv_iter->lv_origin.lr_start,
                              lv_iter->lv_origin.length(),
                              ts_str);
            auto shift = (int) (ts_str.size() - lv_iter->lv_origin.length());
            if (shift != 0) {
                this->lss_token_shifts.emplace_back(lv_iter->lv_origin.lr_start,
                                                    shift);
            }
        }
    }

    std::optional<exttm> adjusted_tm;
    auto time_attr
        = find_string_attr(this->lss_token_al.al_attrs, &L_TIMESTAMP);
    if (!this->lss_token_line->is_continued() && !format->lf_formatted_lines
        && (this->lss_token_file->is_time_adjusted()
            || ((format->lf_timestamp_flags & ETF_ZONE_SET
                 || format->lf_date_time.dts_default_zone != nullptr)
                && format->lf_date_time.dts_zoned_to_local)
            || format->lf_timestamp_flags & ETF_MACHINE_ORIENTED
            || !(format->lf_timestamp_flags & ETF_DAY_SET)
            || !(format->lf_timestamp_flags & ETF_MONTH_SET))
        && format->lf_date_time.dts_fmt_lock != -1)
    {
        if (time_attr != this->lss_token_al.al_attrs.end()) {
            const auto time_range = time_attr->sa_range;
            const auto time_sf
                = string_fragment::from_str_range(this->lss_token_al.al_string,
                                                  time_range.lr_start,
                                                  time_range.lr_end);
            adjusted_tm = format->tm_for_display(this->lss_token_line, time_sf);

            char buffer[128];
            const char* fmt;
            ssize_t len;

            if (format->lf_timestamp_flags & ETF_MACHINE_ORIENTED
                || !(format->lf_timestamp_flags & ETF_DAY_SET)
                || !(format->lf_timestamp_flags & ETF_MONTH_SET))
            {
                if (format->lf_timestamp_flags & ETF_NANOS_SET) {
                    fmt = "%Y-%m-%d %H:%M:%S.%N";
                } else if (format->lf_timestamp_flags & ETF_MICROS_SET) {
                    fmt = "%Y-%m-%d %H:%M:%S.%f";
                } else if (format->lf_timestamp_flags & ETF_MILLIS_SET) {
                    fmt = "%Y-%m-%d %H:%M:%S.%L";
                } else {
                    fmt = "%Y-%m-%d %H:%M:%S";
                }
                len = ftime_fmt(
                    buffer, sizeof(buffer), fmt, adjusted_tm.value());
            } else {
                len = format->lf_date_time.ftime(
                    buffer,
                    sizeof(buffer),
                    format->get_timestamp_formats(),
                    adjusted_tm.value());
            }

            value_out.replace(
                time_range.lr_start, time_range.length(), buffer, len);
            this->lss_token_shifts.emplace_back(
                time_range.lr_start, (int) (len - time_range.length()));
        }
    }

    // Insert space for the file/search-hit markers.
    value_out.insert(0, 1, ' ');
    this->lss_time_column_size = 0;
    if (this->lss_line_context == line_context_t::time_column) {
        if (time_attr != this->lss_token_al.al_attrs.end()) {
            const char* fmt;
            if (this->lss_all_timestamp_flags
                & (ETF_MICROS_SET | ETF_NANOS_SET))
            {
                fmt = "%H:%M:%S.%f";
            } else if (this->lss_all_timestamp_flags & ETF_MILLIS_SET) {
                fmt = "%H:%M:%S.%L";
            } else {
                fmt = "%H:%M:%S";
            }
            if (!adjusted_tm) {
                const auto time_range = time_attr->sa_range;
                const auto time_sf = string_fragment::from_str_range(
                    this->lss_token_al.al_string,
                    time_range.lr_start,
                    time_range.lr_end);
                adjusted_tm
                    = format->tm_for_display(this->lss_token_line, time_sf);
            }
            adjusted_tm->et_flags |= this->lss_all_timestamp_flags
                & (ETF_MILLIS_SET | ETF_MICROS_SET | ETF_NANOS_SET);
            char buffer[128];
            this->lss_time_column_size
                = ftime_fmt(buffer, sizeof(buffer), fmt, adjusted_tm.value());
            if (this->tss_view->is_selectable()
                && this->tss_view->get_selection() == row)
            {
                buffer[this->lss_time_column_size] = ' ';
                buffer[this->lss_time_column_size + 1] = ' ';
                this->lss_time_column_size += 2;
            } else {
                constexpr char block[] = "\u258c ";

                strcpy(&buffer[this->lss_time_column_size], block);
                this->lss_time_column_size += sizeof(block) - 1;
            }
            if (time_attr->sa_range.lr_start != 0) {
                buffer[this->lss_time_column_size] = ' ';
                this->lss_time_column_size += 1;
                this->lss_time_column_padding = 1;
            } else {
                this->lss_time_column_padding = 0;
            }
            value_out.insert(1, buffer, this->lss_time_column_size);
            this->lss_token_al.al_attrs.emplace_back(time_attr->sa_range,
                                                     SA_REPLACED.value());
        }
        if (format->lf_level_hideable) {
            auto level_attr
                = find_string_attr(this->lss_token_al.al_attrs, &L_LEVEL);
            if (level_attr != this->lss_token_al.al_attrs.end()) {
                this->lss_token_al.al_attrs.emplace_back(level_attr->sa_range,
                                                         SA_REPLACED.value());
            }
        }
    } else if (this->lss_line_context < line_context_t::none) {
        size_t file_offset_end;
        std::string name;
        if (this->lss_line_context == line_context_t::filename) {
            file_offset_end = this->lss_filename_width;
            name = fmt::to_string(this->lss_token_file->get_filename());
            if (file_offset_end < name.size()) {
                file_offset_end = name.size();
                this->lss_filename_width = name.size();
            }
        } else {
            file_offset_end = this->lss_basename_width;
            name = fmt::to_string(this->lss_token_file->get_unique_path());
            if (file_offset_end < name.size()) {
                file_offset_end = name.size();
                this->lss_basename_width = name.size();
            }
        }
        value_out.insert(0, file_offset_end - name.size(), ' ');
        value_out.insert(0, name);
    }

    if (this->tas_display_time_offset) {
        auto row_vl = vis_line_t(row);
        auto relstr = this->get_time_offset_for_line(tc, row_vl);
        value_out = fmt::format(FMT_STRING("{: >12}|{}"), relstr, value_out);
    }

    this->lss_in_value_for_line = false;

    return retval;
}

void
logfile_sub_source::text_attrs_for_line(textview_curses& lv,
                                        int row,
                                        string_attrs_t& value_out)
{
    if (this->lss_indexing_in_progress) {
        return;
    }

    auto& vc = view_colors::singleton();
    logline* next_line = nullptr;
    line_range lr;
    int time_offset_end = 0;
    text_attrs attrs;
    auto* format = this->lss_token_file->get_format_ptr();

    if ((row + 1) < (int) this->lss_filtered_index.size()) {
        next_line = this->find_line(this->at(vis_line_t(row + 1)));
    }

    if (next_line != nullptr
        && (day_num(next_line->get_time<std::chrono::seconds>().count())
            > day_num(this->lss_token_line->get_time<std::chrono::seconds>()
                          .count())))
    {
        attrs |= text_attrs::style::underline;
    }

    const auto& line_values = this->lss_token_values;

    lr.lr_start = 0;
    lr.lr_end = -1;
    this->lss_token_al.al_attrs.emplace_back(
        lr, SA_LEVEL.value(this->lss_token_line->get_msg_level()));

    lr.lr_start = time_offset_end;
    lr.lr_end = -1;

    if (!attrs.empty()) {
        this->lss_token_al.al_attrs.emplace_back(lr, VC_STYLE.value(attrs));
    }

    if (this->lss_token_line->get_msg_level() == log_level_t::LEVEL_INVALID) {
        for (auto& token_attr : this->lss_token_al.al_attrs) {
            if (token_attr.sa_type != &SA_INVALID) {
                continue;
            }

            this->lss_token_al.al_attrs.emplace_back(
                token_attr.sa_range, VC_ROLE.value(role_t::VCR_INVALID_MSG));
        }
    }

    for (const auto& line_value : line_values.lvv_values) {
        if ((!(this->lss_token_flags & RF_FULL)
             && line_value.lv_sub_offset
                 != this->lss_token_line->get_sub_offset())
            || !line_value.lv_origin.is_valid())
        {
            continue;
        }

        if (line_value.lv_meta.is_hidden()) {
            this->lss_token_al.al_attrs.emplace_back(
                line_value.lv_origin, SA_HIDDEN.value(ui_icon_t::hidden));
        }

        if (!line_value.lv_meta.lvm_identifier
            || !line_value.lv_origin.is_valid())
        {
            continue;
        }

        if (line_value.lv_highlighted) {
            continue;
        }

        this->lss_token_al.al_attrs.emplace_back(
            line_value.lv_origin, VC_ROLE.value(role_t::VCR_IDENTIFIER));
    }

    // Apply shifts right-to-left so positions remain valid.
    std::stable_sort(
        this->lss_token_shifts.begin(),
        this->lss_token_shifts.end(),
        [](const auto& a, const auto& b) { return a.first > b.first; });
    for (const auto& shift : this->lss_token_shifts) {
        shift_string_attrs(
            this->lss_token_al.al_attrs, shift.first + 1, shift.second);
    }

    shift_string_attrs(this->lss_token_al.al_attrs, 0, 1);

    lr.lr_start = 0;
    lr.lr_end = 1;
    {
        auto& bm = lv.get_bookmarks();
        const auto& bv = bm[&BM_FILES];
        bool is_first_for_file = bv.bv_tree.exists(vis_line_t(row));
        bool is_last_for_file = bv.bv_tree.exists(vis_line_t(row + 1));
        auto graph = NCACS_VLINE;
        if (is_first_for_file) {
            if (is_last_for_file) {
                graph = NCACS_HLINE;
            } else {
                graph = NCACS_ULCORNER;
            }
        } else if (is_last_for_file) {
            graph = NCACS_LLCORNER;
        }
        this->lss_token_al.al_attrs.emplace_back(lr, VC_GRAPHIC.value(graph));

        if (!(this->lss_token_flags & RF_FULL)) {
            const auto& bv_search = bm[&textview_curses::BM_SEARCH];

            if (bv_search.bv_tree.exists(vis_line_t(row))) {
                lr.lr_start = 0;
                lr.lr_end = 1;
                this->lss_token_al.al_attrs.emplace_back(
                    lr, VC_STYLE.value(text_attrs::with_reverse()));
            }
        }
    }

    this->lss_token_al.al_attrs.emplace_back(
        lr,
        VC_STYLE.value(
            vc.attrs_for_ident(this->lss_token_file->get_filename())));

    if (this->lss_line_context < line_context_t::none) {
        size_t file_offset_end
            = (this->lss_line_context == line_context_t::filename)
            ? this->lss_filename_width
            : this->lss_basename_width;

        shift_string_attrs(this->lss_token_al.al_attrs, 0, file_offset_end);

        lr.lr_start = 0;
        lr.lr_end = file_offset_end + 1;
        this->lss_token_al.al_attrs.emplace_back(
            lr,
            VC_STYLE.value(
                vc.attrs_for_ident(this->lss_token_file->get_filename())));
    } else if (this->lss_time_column_size > 0) {
        shift_string_attrs(
            this->lss_token_al.al_attrs, 1, this->lss_time_column_size);

        ui_icon_t icon;
        switch (this->lss_token_line->get_msg_level()) {
            case LEVEL_TRACE:
                icon = ui_icon_t::log_level_trace;
                break;
            case LEVEL_DEBUG:
            case LEVEL_DEBUG2:
            case LEVEL_DEBUG3:
            case LEVEL_DEBUG4:
            case LEVEL_DEBUG5:
                icon = ui_icon_t::log_level_debug;
                break;
            case LEVEL_INFO:
                icon = ui_icon_t::log_level_info;
                break;
            case LEVEL_STATS:
                icon = ui_icon_t::log_level_stats;
                break;
            case LEVEL_NOTICE:
                icon = ui_icon_t::log_level_notice;
                break;
            case LEVEL_WARNING:
                icon = ui_icon_t::log_level_warning;
                break;
            case LEVEL_ERROR:
                icon = ui_icon_t::log_level_error;
                break;
            case LEVEL_CRITICAL:
                icon = ui_icon_t::log_level_critical;
                break;
            case LEVEL_FATAL:
                icon = ui_icon_t::log_level_fatal;
                break;
            default:
                icon = ui_icon_t::hidden;
                break;
        }
        auto extra_space_size = this->lss_time_column_padding;
        lr.lr_start = 1 + this->lss_time_column_size - 1 - extra_space_size;
        lr.lr_end = 1 + this->lss_time_column_size - extra_space_size;
        this->lss_token_al.al_attrs.emplace_back(lr, VC_ICON.value(icon));
        if (this->tss_view->is_selectable()
            && this->tss_view->get_selection() != row)
        {
            lr.lr_start = 1;
            lr.lr_end = 1 + this->lss_time_column_size - 2 - extra_space_size;
            this->lss_token_al.al_attrs.emplace_back(
                lr, VC_ROLE.value(role_t::VCR_TIME_COLUMN));
            if (this->lss_token_line->is_time_skewed()) {
                this->lss_token_al.al_attrs.emplace_back(
                    lr, VC_ROLE.value(role_t::VCR_SKEWED_TIME));
            }
            lr.lr_start = 1 + this->lss_time_column_size - 2 - extra_space_size;
            lr.lr_end = 1 + this->lss_time_column_size - 1 - extra_space_size;
            this->lss_token_al.al_attrs.emplace_back(
                lr, VC_ROLE.value(role_t::VCR_TIME_COLUMN_TO_TEXT));
        }
    }

    if (this->tas_display_time_offset) {
        time_offset_end = 13;
        lr.lr_start = 0;
        lr.lr_end = time_offset_end;

        shift_string_attrs(this->lss_token_al.al_attrs, 0, time_offset_end);

        this->lss_token_al.al_attrs.emplace_back(
            lr, VC_ROLE.value(role_t::VCR_OFFSET_TIME));
        this->lss_token_al.al_attrs.emplace_back(line_range(12, 13),
                                                 VC_GRAPHIC.value(NCACS_VLINE));

        auto bar_role = role_t::VCR_NONE;

        switch (this->get_line_accel_direction(vis_line_t(row))) {
            case log_accel::direction_t::A_STEADY:
                break;
            case log_accel::direction_t::A_DECEL:
                bar_role = role_t::VCR_DIFF_DELETE;
                break;
            case log_accel::direction_t::A_ACCEL:
                bar_role = role_t::VCR_DIFF_ADD;
                break;
        }
        if (bar_role != role_t::VCR_NONE) {
            this->lss_token_al.al_attrs.emplace_back(line_range(12, 13),
                                                     VC_ROLE.value(bar_role));
        }
    }

    lr.lr_start = 0;
    lr.lr_end = -1;
    this->lss_token_al.al_attrs.emplace_back(
        lr, L_FILE.value(this->lss_token_file));
    this->lss_token_al.al_attrs.emplace_back(
        lr, SA_FORMAT.value(format->get_name()));

    {
        auto line_meta_context = this->get_bookmark_metadata_context(
            vis_line_t(row + 1), bookmark_metadata::categories::partition);
        if (line_meta_context.bmc_current_metadata) {
            lr.lr_start = 0;
            lr.lr_end = -1;
            this->lss_token_al.al_attrs.emplace_back(
                lr,
                L_PARTITION.value(
                    line_meta_context.bmc_current_metadata.value()));
        }

        auto line_meta_opt = this->find_bookmark_metadata(vis_line_t(row));

        if (line_meta_opt) {
            lr.lr_start = 0;
            lr.lr_end = -1;
            this->lss_token_al.al_attrs.emplace_back(
                lr, L_META.value(line_meta_opt.value()));
        }
    }

    if (this->lss_time_column_size == 0) {
        if (this->lss_token_file->is_time_adjusted()) {
            auto time_range = find_string_attr_range(
                this->lss_token_al.al_attrs, &L_TIMESTAMP);

            if (time_range.lr_end != -1) {
                this->lss_token_al.al_attrs.emplace_back(
                    time_range, VC_ROLE.value(role_t::VCR_ADJUSTED_TIME));
            }
        } else if (this->lss_token_line->is_time_skewed()) {
            auto time_range = find_string_attr_range(
                this->lss_token_al.al_attrs, &L_TIMESTAMP);

            if (time_range.lr_end != -1) {
                this->lss_token_al.al_attrs.emplace_back(
                    time_range, VC_ROLE.value(role_t::VCR_SKEWED_TIME));
            }
        }
    }

    if (this->tss_preview_min_log_level
        && this->lss_token_line->get_msg_level()
            < this->tss_preview_min_log_level)
    {
        auto color = styling::color_unit::from_palette(
            lnav::enums::to_underlying(ansi_color::red));
        this->lss_token_al.al_attrs.emplace_back(line_range{0, 1},
                                                 VC_BACKGROUND.value(color));
    }
    if (this->ttt_preview_min_time
        && this->lss_token_line->get_timeval() < this->ttt_preview_min_time)
    {
        auto color = styling::color_unit::from_palette(
            lnav::enums::to_underlying(ansi_color::red));
        this->lss_token_al.al_attrs.emplace_back(line_range{0, 1},
                                                 VC_BACKGROUND.value(color));
    }
    if (this->ttt_preview_max_time
        && this->ttt_preview_max_time < this->lss_token_line->get_timeval())
    {
        auto color = styling::color_unit::from_palette(
            lnav::enums::to_underlying(ansi_color::red));
        this->lss_token_al.al_attrs.emplace_back(line_range{0, 1},
                                                 VC_BACKGROUND.value(color));
    }
    if (!this->lss_token_line->is_continued()) {
        if (this->lss_preview_filter_stmt != nullptr) {
            auto color = styling::color_unit::EMPTY;
            auto eval_res
                = this->eval_sql_filter(this->lss_preview_filter_stmt.in(),
                                        this->lss_token_file_data,
                                        this->lss_token_line);
            if (eval_res.isErr()) {
                color = palette_color{
                    lnav::enums::to_underlying(ansi_color::yellow)};
                this->lss_token_al.al_attrs.emplace_back(
                    line_range{0, -1},
                    SA_ERROR.value(
                        eval_res.unwrapErr().to_attr_line().get_string()));
            } else {
                auto matched = eval_res.unwrap();

                if (matched) {
                    color = palette_color{
                        lnav::enums::to_underlying(ansi_color::green)};
                } else {
                    color = palette_color{
                        lnav::enums::to_underlying(ansi_color::red)};
                    this->lss_token_al.al_attrs.emplace_back(
                        line_range{0, 1},
                        VC_STYLE.value(text_attrs::with_blink()));
                }
            }
            this->lss_token_al.al_attrs.emplace_back(
                line_range{0, 1}, VC_BACKGROUND.value(color));
        }

        auto sql_filter_opt = this->get_sql_filter();
        if (sql_filter_opt) {
            auto* sf = (sql_filter*) sql_filter_opt.value().get();
            auto eval_res = this->eval_sql_filter(sf->sf_filter_stmt.in(),
                                                  this->lss_token_file_data,
                                                  this->lss_token_line);
            if (eval_res.isErr()) {
                auto msg = fmt::format(
                    FMT_STRING(
                        "filter expression evaluation failed with -- {}"),
                    eval_res.unwrapErr().to_attr_line().get_string());
                auto cu = styling::color_unit::from_palette(palette_color{
                    lnav::enums::to_underlying(ansi_color::yellow)});
                this->lss_token_al.al_attrs.emplace_back(line_range{0, -1},
                                                         SA_ERROR.value(msg));
                this->lss_token_al.al_attrs.emplace_back(
                    line_range{0, 1}, VC_BACKGROUND.value(cu));
            }
        }
    }

    value_out = std::move(this->lss_token_al.al_attrs);
}

struct logline_cmp {
    explicit logline_cmp(logfile_sub_source& lc) : llss_controller(lc) {}

    bool operator()(const logfile_sub_source::indexed_content& lhs,
                    const logfile_sub_source::indexed_content& rhs) const
    {
        const auto* ll_lhs = this->llss_controller.find_line(lhs.value());
        const auto* ll_rhs = this->llss_controller.find_line(rhs.value());

        return (*ll_lhs) < (*ll_rhs);
    }

    bool operator()(const logfile_sub_source::indexed_content& lhs,
                    const timeval& rhs) const
    {
        const auto* ll_lhs = this->llss_controller.find_line(lhs.value());

        return *ll_lhs < rhs;
    }

    logfile_sub_source& llss_controller;
};

logfile_sub_source::rebuild_result
logfile_sub_source::rebuild_index(std::optional<ui_clock::time_point> deadline)
{
    if (this->tss_view == nullptr) {
        return rebuild_result::rr_no_change;
    }

    this->lss_indexing_in_progress = true;
    auto fin = finally([this]() { this->lss_indexing_in_progress = false; });

    iterator iter;
    size_t total_lines = 0;
    size_t est_remaining_lines = 0;
    auto all_time_ordered_formats = true;
    auto full_sort = this->lss_index.empty();
    int file_count = 0;
    auto force = std::exchange(this->lss_force_rebuild, false);
    auto retval = rebuild_result::rr_no_change;
    std::optional<timeval> lowest_tv = std::nullopt;
    auto search_start = 0_vl;

    if (force) {
        log_debug("forced to full rebuild");
        retval = rebuild_result::rr_full_rebuild;
        full_sort = true;
        this->tss_level_filtered_count = 0;
        this->lss_index.clear();
    }

    std::vector<size_t> file_order(this->lss_files.size());

    for (size_t lpc = 0; lpc < file_order.size(); lpc++) {
        file_order[lpc] = lpc;
    }
    if (!this->lss_index.empty()) {
        std::stable_sort(file_order.begin(),
                         file_order.end(),
                         [this](const auto& left, const auto& right) {
                             const auto& left_ld = this->lss_files[left];
                             const auto& right_ld = this->lss_files[right];

                             if (left_ld->get_file_ptr() == nullptr) {
                                 return true;
                             }
                             if (right_ld->get_file_ptr() == nullptr) {
                                 return false;
                             }

                             return left_ld->get_file_ptr()->back()
                                 < right_ld->get_file_ptr()->back();
                         });
    }

    bool time_left = true;
    this->lss_all_timestamp_flags = 0;
    for (const auto file_index : file_order) {
        auto& ld = *(this->lss_files[file_index]);
        auto* lf = ld.get_file_ptr();

        if (lf == nullptr) {
            if (ld.ld_lines_indexed > 0) {
                log_debug("%zu: file closed, doing full rebuild",
                          ld.ld_file_index);
                force = true;
                retval = rebuild_result::rr_full_rebuild;
                full_sort = true;
            }
        } else {
            if (!lf->get_format_ptr()->lf_time_ordered) {
                all_time_ordered_formats = false;
            }
            if (time_left && deadline && ui_clock::now() > deadline.value()) {
                log_debug("no time left, skipping %s",
                          lf->get_filename_as_string().c_str());
                time_left = false;
            }
            this->lss_all_timestamp_flags
                |= lf->get_format_ptr()->lf_timestamp_flags;

            if (!this->tss_view->is_paused() && time_left) {
                auto log_rebuild_res = lf->rebuild_index(deadline);

                if (ld.ld_lines_indexed < lf->size()
                    && log_rebuild_res
                        == logfile::rebuild_result_t::NO_NEW_LINES)
                {
                    // This is a bit awkward... if the logfile indexing was
                    // complete before being added to us, we need to adjust
                    // the rebuild result to make it look like new lines
                    // were added.
                    log_rebuild_res = logfile::rebuild_result_t::NEW_LINES;
                }
                switch (log_rebuild_res) {
                    case logfile::rebuild_result_t::NO_NEW_LINES:
                        break;
                    case logfile::rebuild_result_t::NEW_LINES:
                        if (retval == rebuild_result::rr_no_change) {
                            retval = rebuild_result::rr_appended_lines;
                        }
                        log_debug("new lines for %s:%zu",
                                  lf->get_filename_as_string().c_str(),
                                  lf->size());
                        if (!this->lss_index.empty()
                            && lf->size() > ld.ld_lines_indexed)
                        {
                            auto& new_file_line = (*lf)[ld.ld_lines_indexed];
                            auto cl = this->lss_index.back().value();
                            auto* last_indexed_line = this->find_line(cl);

                            // If there are new lines that are older than what
                            // we have in the index, we need to resort.
                            if (last_indexed_line == nullptr
                                || new_file_line
                                    < last_indexed_line->get_timeval())
                            {
                                log_debug(
                                    "%s:%ld: found older lines, full "
                                    "rebuild: %p  %lld < %lld",
                                    lf->get_filename().c_str(),
                                    ld.ld_lines_indexed,
                                    last_indexed_line,
                                    new_file_line
                                        .get_time<std::chrono::microseconds>()
                                        .count(),
                                    last_indexed_line == nullptr
                                        ? (uint64_t) -1
                                        : last_indexed_line
                                              ->get_time<
                                                  std::chrono::microseconds>()
                                              .count());
                                if (retval <= rebuild_result::rr_partial_rebuild
                                    && all_time_ordered_formats)
                                {
                                    retval = rebuild_result::rr_partial_rebuild;
                                    if (!lowest_tv
                                        || new_file_line.get_timeval()
                                            < lowest_tv.value())
                                    {
                                        lowest_tv = new_file_line.get_timeval();
                                    }
                                } else {
                                    log_debug(
                                        "already doing full rebuild, doing "
                                        "full_sort as well");
                                    force = true;
                                    full_sort = true;
                                }
                            }
                        }
                        break;
                    case logfile::rebuild_result_t::INVALID:
                    case logfile::rebuild_result_t::NEW_ORDER:
                        log_debug("%s: log file has a new order, full rebuild",
                                  lf->get_filename().c_str());
                        retval = rebuild_result::rr_full_rebuild;
                        force = true;
                        full_sort = true;
                        break;
                }
            }
            file_count += 1;
            total_lines += lf->size();

            est_remaining_lines += lf->estimated_remaining_lines();
        }
    }

    if (!all_time_ordered_formats
        && retval == rebuild_result::rr_partial_rebuild)
    {
        force = true;
        full_sort = true;
        retval = rebuild_result::rr_full_rebuild;
    }

    if (this->lss_index.reserve(total_lines + est_remaining_lines)) {
        // The index array was reallocated, just do a full sort/rebuild since
        // it's been cleared out.
        log_debug("expanding index capacity %zu", this->lss_index.ba_capacity);
        force = true;
        retval = rebuild_result::rr_full_rebuild;
        full_sort = true;
        this->tss_level_filtered_count = 0;
    }

    auto& vis_bm = this->tss_view->get_bookmarks();

    if (force) {
        for (iter = this->lss_files.begin(); iter != this->lss_files.end();
             iter++)
        {
            (*iter)->ld_lines_indexed = 0;
        }

        this->lss_index.clear();
        this->lss_filtered_index.clear();
        this->tss_level_filtered_count = 0;
        this->lss_longest_line = 0;
        this->lss_basename_width = 0;
        this->lss_filename_width = 0;
        vis_bm[&textview_curses::BM_USER_EXPR].clear();
        if (this->lss_index_delegate) {
            this->lss_index_delegate->index_start(*this);
        }
    } else if (retval == rebuild_result::rr_partial_rebuild) {
        size_t remaining = 0;

        log_debug("partial rebuild with lowest time: %ld",
                  lowest_tv.value().tv_sec);
        for (iter = this->lss_files.begin(); iter != this->lss_files.end();
             iter++)
        {
            logfile_data& ld = *(*iter);
            auto* lf = ld.get_file_ptr();

            if (lf == nullptr) {
                continue;
            }

            require(lf->get_format_ptr()->lf_time_ordered);

            auto line_iter = lf->find_from_time(lowest_tv.value());

            if (line_iter) {
                log_debug("lowest line time %ld; line %ld; size %ld; path=%s",
                          line_iter.value()->get_timeval().tv_sec,
                          std::distance(lf->cbegin(), line_iter.value()),
                          lf->size(),
                          lf->get_filename_as_string().c_str());
            }
            ld.ld_lines_indexed
                = std::distance(lf->cbegin(), line_iter.value_or(lf->cend()));
            remaining += lf->size() - ld.ld_lines_indexed;
        }

        auto* row_iter = std::lower_bound(this->lss_index.begin(),
                                          this->lss_index.end(),
                                          lowest_tv.value(),
                                          logline_cmp(*this));
        this->lss_index.shrink_to(
            std::distance(this->lss_index.begin(), row_iter));
        log_debug("new index size %ld/%ld; remain %ld",
                  this->lss_index.ba_size,
                  this->lss_index.ba_capacity,
                  remaining);
        auto filt_row_iter = std::lower_bound(this->lss_filtered_index.begin(),
                                              this->lss_filtered_index.end(),
                                              lowest_tv.value(),
                                              filtered_logline_cmp(*this));
        this->lss_filtered_index.resize(
            std::distance(this->lss_filtered_index.begin(), filt_row_iter));
        search_start = vis_line_t(this->lss_filtered_index.size());

        if (this->lss_index_delegate) {
            this->lss_index_delegate->index_start(*this);
            for (const auto row_in_full_index : this->lss_filtered_index) {
                auto cl = this->lss_index[row_in_full_index].value();
                uint64_t line_number;
                auto ld_iter = this->find_data(cl, line_number);
                auto& ld = *ld_iter;
                auto line_iter = ld->get_file_ptr()->begin() + line_number;

                this->lss_index_delegate->index_line(
                    *this, ld->get_file_ptr(), line_iter);
            }
        }
    }

    if (this->lss_index.empty() && !time_left) {
        log_info("ran out of time, skipping rebuild");
        // need to make sure we rebuild in case no new data comes in
        this->lss_force_rebuild = true;
        return rebuild_result::rr_appended_lines;
    }

    if (retval != rebuild_result::rr_no_change || force) {
        size_t index_size = 0, start_size = this->lss_index.size();
        logline_cmp line_cmper(*this);

        for (auto& ld : this->lss_files) {
            auto* lf = ld->get_file_ptr();

            if (lf == nullptr) {
                continue;
            }
            this->lss_longest_line = std::max(
                this->lss_longest_line, lf->get_longest_line_length() + 1);
            this->lss_basename_width
                = std::max(this->lss_basename_width,
                           lf->get_unique_path().native().size());
            this->lss_filename_width = std::max(
                this->lss_filename_width, lf->get_filename().native().size());
        }

        if (full_sort) {
            log_trace("rebuild_index full sort");
            for (auto& ld : this->lss_files) {
                auto* lf = ld->get_file_ptr();

                if (lf == nullptr) {
                    continue;
                }

                for (size_t line_index = 0; line_index < lf->size();
                     line_index++)
                {
                    const auto lf_iter
                        = ld->get_file_ptr()->begin() + line_index;
                    if (lf_iter->is_ignored()) {
                        continue;
                    }

                    content_line_t con_line(
                        ld->ld_file_index * MAX_LINES_PER_FILE + line_index);

                    if (lf_iter->is_meta_marked()) {
                        auto start_iter = lf_iter;
                        while (start_iter->is_continued()) {
                            --start_iter;
                        }
                        int start_index
                            = start_iter - ld->get_file_ptr()->begin();
                        content_line_t start_con_line(ld->ld_file_index
                                                          * MAX_LINES_PER_FILE
                                                      + start_index);

                        auto& line_meta
                            = ld->get_file_ptr()
                                  ->get_bookmark_metadata()[start_index];
                        if (line_meta.has(bookmark_metadata::categories::notes))
                        {
                            this->lss_user_marks[&textview_curses::BM_META]
                                .insert_once(start_con_line);
                        }
                        if (line_meta.has(
                                bookmark_metadata::categories::partition))
                        {
                            this->lss_user_marks[&textview_curses::BM_PARTITION]
                                .insert_once(start_con_line);
                        }
                    }
                    this->lss_index.push_back(
                        indexed_content{con_line, lf_iter});
                }
            }

            // XXX get rid of this full sort on the initial run, it's not
            // needed unless the file is not in time-order
            if (this->lss_sorting_observer) {
                this->lss_sorting_observer(*this, 0, this->lss_index.size());
            }
            std::sort(
                this->lss_index.begin(), this->lss_index.end(), line_cmper);
            if (this->lss_sorting_observer) {
                this->lss_sorting_observer(
                    *this, this->lss_index.size(), this->lss_index.size());
            }
        } else {
            kmerge_tree_c<logline, logfile_data, logfile::iterator> merge(
                file_count);

            for (iter = this->lss_files.begin(); iter != this->lss_files.end();
                 iter++)
            {
                auto* ld = iter->get();
                auto* lf = ld->get_file_ptr();
                if (lf == nullptr) {
                    continue;
                }

                merge.add(ld, lf->begin() + ld->ld_lines_indexed, lf->end());
                index_size += lf->size();
            }

            file_off_t index_off = 0;
            merge.execute();
            if (this->lss_sorting_observer) {
                this->lss_sorting_observer(*this, index_off, index_size);
            }
            log_trace("k-way merge");
            for (;;) {
                logfile::iterator lf_iter;
                logfile_data* ld;

                if (!merge.get_top(ld, lf_iter)) {
                    break;
                }

                if (!lf_iter->is_ignored()) {
                    int file_index = ld->ld_file_index;
                    int line_index = lf_iter - ld->get_file_ptr()->begin();

                    content_line_t con_line(file_index * MAX_LINES_PER_FILE
                                            + line_index);

                    if (lf_iter->is_meta_marked()) {
                        auto start_iter = lf_iter;
                        while (start_iter->is_continued()) {
                            --start_iter;
                        }
                        int start_index
                            = start_iter - ld->get_file_ptr()->begin();
                        content_line_t start_con_line(
                            file_index * MAX_LINES_PER_FILE + start_index);

                        auto& line_meta
                            = ld->get_file_ptr()
                                  ->get_bookmark_metadata()[start_index];
                        if (line_meta.has(bookmark_metadata::categories::notes))
                        {
                            this->lss_user_marks[&textview_curses::BM_META]
                                .insert_once(start_con_line);
                        }
                        if (line_meta.has(
                                bookmark_metadata::categories::partition))
                        {
                            this->lss_user_marks[&textview_curses::BM_PARTITION]
                                .insert_once(start_con_line);
                        }
                    }
                    this->lss_index.push_back(
                        indexed_content{con_line, lf_iter});
                }

                merge.next();
                index_off += 1;
                if (index_off % 100000 == 0 && this->lss_sorting_observer) {
                    this->lss_sorting_observer(*this, index_off, index_size);
                }
            }
            if (this->lss_sorting_observer) {
                this->lss_sorting_observer(*this, index_size, index_size);
            }
        }

        for (iter = this->lss_files.begin(); iter != this->lss_files.end();
             ++iter)
        {
            auto* lf = (*iter)->get_file_ptr();

            if (lf == nullptr) {
                continue;
            }

            (*iter)->ld_lines_indexed = lf->size();
        }

        this->lss_filtered_index.reserve(this->lss_index.size());

        uint32_t filter_in_mask, filter_out_mask;
        this->get_filters().get_enabled_mask(filter_in_mask, filter_out_mask);

        if (start_size == 0 && this->lss_index_delegate != nullptr) {
            this->lss_index_delegate->index_start(*this);
        }

        log_trace("filtered index");
        for (size_t index_index = start_size;
             index_index < this->lss_index.size();
             index_index++)
        {
            const auto cl = this->lss_index[index_index].value();
            uint64_t line_number;
            auto ld = this->find_data(cl, line_number);

            if (!(*ld)->is_visible()) {
                continue;
            }

            auto* lf = (*ld)->get_file_ptr();
            auto line_iter = lf->begin() + line_number;

            if (line_iter->is_ignored()) {
                continue;
            }

            if (!this->tss_apply_filters
                || (!(*ld)->ld_filter_state.excluded(
                        filter_in_mask, filter_out_mask, line_number)
                    && this->check_extra_filters(ld, line_iter)))
            {
                auto eval_res = this->eval_sql_filter(
                    this->lss_marker_stmt.in(), ld, line_iter);
                if (eval_res.isErr()) {
                    line_iter->set_expr_mark(false);
                } else {
                    auto matched = eval_res.unwrap();

                    line_iter->set_expr_mark(matched);
                    if (matched) {
                        vis_bm[&textview_curses::BM_USER_EXPR].insert_once(
                            vis_line_t(this->lss_filtered_index.size()));
                        this->lss_user_marks[&textview_curses::BM_USER_EXPR]
                            .insert_once(cl);
                    }
                }
                this->lss_filtered_index.push_back(index_index);
                if (this->lss_index_delegate != nullptr) {
                    this->lss_index_delegate->index_line(*this, lf, line_iter);
                }
            }
        }

        this->lss_indexing_in_progress = false;

        if (this->lss_index_delegate != nullptr) {
            this->lss_index_delegate->index_complete(*this);
        }
    }

    switch (retval) {
        case rebuild_result::rr_no_change:
            break;
        case rebuild_result::rr_full_rebuild:
            log_debug("redoing search");
            this->lss_index_generation += 1;
            this->tss_view->reload_data();
            this->tss_view->redo_search();
            break;
        case rebuild_result::rr_partial_rebuild:
            log_debug("redoing search from: %d", (int) search_start);
            this->lss_index_generation += 1;
            this->tss_view->reload_data();
            this->tss_view->search_new_data(search_start);
            break;
        case rebuild_result::rr_appended_lines:
            this->tss_view->reload_data();
            this->tss_view->search_new_data();
            break;
    }

    return retval;
}

void
logfile_sub_source::text_update_marks(vis_bookmarks& bm)
{
    logfile* last_file = nullptr;
    vis_line_t vl;

    auto& bm_warnings = bm[&textview_curses::BM_WARNINGS];
    auto& bm_errors = bm[&textview_curses::BM_ERRORS];
    auto& bm_files = bm[&BM_FILES];

    bm_warnings.clear();
    bm_errors.clear();
    bm_files.clear();

    std::vector<const bookmark_type_t*> used_marks;
    for (const auto* bmt :
         {
             &textview_curses::BM_USER,
             &textview_curses::BM_USER_EXPR,
             &textview_curses::BM_PARTITION,
             &textview_curses::BM_META,
             &textview_curses::BM_STICKY,
         })
    {
        bm[bmt].clear();
        if (!this->lss_user_marks[bmt].empty()) {
            used_marks.emplace_back(bmt);
        }
    }

    for (; vl < (int) this->lss_filtered_index.size(); ++vl) {
        const auto& orig_ic = this->lss_index[this->lss_filtered_index[vl]];
        auto cl = orig_ic.value();
        auto* lf = this->find_file_ptr(cl);

        for (const auto& bmt : used_marks) {
            auto& user_mark = this->lss_user_marks[bmt];
            if (user_mark.bv_tree.exists(orig_ic.value())) {
                bm[bmt].insert_once(vl);
            }
        }

        if (lf != last_file) {
            bm_files.insert_once(vl);
        }

        switch (orig_ic.level()) {
            case indexed_content::level_t::warning:
                bm_warnings.insert_once(vl);
                break;

            case indexed_content::level_t::error:
                bm_errors.insert_once(vl);
                break;

            default:
                break;
        }

        last_file = lf;
    }
}

void
logfile_sub_source::text_filters_changed()
{
    static auto op = lnav_operation{"text_filters_changed"};

    auto op_guard = lnav_opid_guard::internal(op);
    this->lss_index_generation += 1;
    this->tss_level_filtered_count = 0;

    if (this->lss_line_meta_changed) {
        this->invalidate_sql_filter();
        this->lss_line_meta_changed = false;
    }

    log_debug("filtering files");
    for (auto& ld : *this) {
        auto* lf = ld->get_file_ptr();

        if (lf != nullptr) {
            ld->ld_filter_state.clear_deleted_filter_state();
            lf->reobserve_from(lf->begin()
                               + ld->ld_filter_state.get_min_count(lf->size()));
        }
    }

    if (this->lss_force_rebuild) {
        log_debug("skipping update since in the middle of force rebuild");
        return;
    }

    auto& vis_bm = this->tss_view->get_bookmarks();
    uint32_t filtered_in_mask, filtered_out_mask;

    this->get_filters().get_enabled_mask(filtered_in_mask, filtered_out_mask);

    if (this->lss_index_delegate != nullptr) {
        this->lss_index_delegate->index_start(*this);
    }
    vis_bm[&textview_curses::BM_USER_EXPR].clear();

    this->lss_filtered_index.clear();
    for (size_t index_index = 0; index_index < this->lss_index.size();
         index_index++)
    {
        auto cl = this->lss_index[index_index].value();
        uint64_t line_number;
        auto ld = this->find_data(cl, line_number);

        if (!(*ld)->is_visible()) {
            continue;
        }

        auto lf = (*ld)->get_file_ptr();
        auto line_iter = lf->begin() + line_number;

        if (!this->tss_apply_filters
            || (!(*ld)->ld_filter_state.excluded(
                    filtered_in_mask, filtered_out_mask, line_number)
                && this->check_extra_filters(ld, line_iter)))
        {
            auto eval_res = this->eval_sql_filter(
                this->lss_marker_stmt.in(), ld, line_iter);
            if (eval_res.isErr()) {
                line_iter->set_expr_mark(false);
            } else {
                auto matched = eval_res.unwrap();

                line_iter->set_expr_mark(matched);
                if (matched) {
                    vis_bm[&textview_curses::BM_USER_EXPR].insert_once(
                        vis_line_t(this->lss_filtered_index.size()));
                    this->lss_user_marks[&textview_curses::BM_USER_EXPR]
                        .insert_once(cl);
                }
            }
            this->lss_filtered_index.push_back(index_index);
            if (this->lss_index_delegate != nullptr) {
                this->lss_index_delegate->index_line(*this, lf, line_iter);
            }
        }
    }

    if (this->lss_index_delegate != nullptr) {
        this->lss_index_delegate->index_complete(*this);
    }

    if (this->tss_view != nullptr) {
        log_debug("reloading view");
        this->tss_view->reload_data();
        this->tss_view->redo_search();
    }
    log_debug("finished filter update");
}

std::optional<json_string>
logfile_sub_source::text_row_details(const textview_curses& tc)
{
    if (this->lss_index.empty()) {
        log_trace("logfile_sub_source::text_row_details empty");
        return std::nullopt;
    }

    auto ov_sel = tc.get_overlay_selection();
    if (ov_sel.has_value()) {
        auto* fos
            = dynamic_cast<field_overlay_source*>(tc.get_overlay_source());
        auto iter = fos->fos_row_to_field_meta.find(ov_sel.value());
        if (iter != fos->fos_row_to_field_meta.end()) {
            auto find_res = this->find_line_with_file(tc.get_top());
            if (find_res) {
                yajlpp_gen gen;

                {
                    yajlpp_map root(gen);

                    root.gen("value");
                    root.gen(iter->second.ri_value);
                }

                return json_string(gen);
            }
        }
    }

    return std::nullopt;
}

bool
logfile_sub_source::list_input_handle_key(listview_curses& lv,
                                          const ncinput& ch)
{
    switch (ch.eff_text[0]) {
        case ' ': {
            auto ov_vl = lv.get_overlay_selection();
            if (ov_vl) {
                auto* fos = dynamic_cast<field_overlay_source*>(
                    lv.get_overlay_source());
                auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
                if (iter != fos->fos_row_to_field_meta.end()
                    && iter->second.ri_meta)
                {
                    auto find_res = this->find_line_with_file(lv.get_top());
                    if (find_res) {
                        auto file_and_line = find_res.value();
                        auto* format = file_and_line.first->get_format_ptr();
                        auto fstates = format->get_field_states();
                        auto state_iter
                            = fstates.find(iter->second.ri_meta->lvm_name);
                        if (state_iter != fstates.end()) {
                            format->hide_field(iter->second.ri_meta->lvm_name,
                                               !state_iter->second.is_hidden());
                            lv.set_needs_update();
                        }
                    }
                }
                return true;
            }
            return false;
        }
        case '#': {
            auto ov_vl = lv.get_overlay_selection();
            if (ov_vl) {
                auto* fos = dynamic_cast<field_overlay_source*>(
                    lv.get_overlay_source());
                auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
                if (iter != fos->fos_row_to_field_meta.end()
                    && iter->second.ri_meta)
                {
                    const auto& meta = iter->second.ri_meta.value();
                    std::string cmd;

                    switch (meta.to_chart_type()) {
                        case chart_type_t::none:
                            break;
                        case chart_type_t::hist: {
                            auto prql = fmt::format(
                                FMT_STRING(
                                    "from {} | stats.hist {} slice:'1h'"),
                                meta.lvm_format.value()->get_name(),
                                meta.lvm_name);
                            cmd = fmt::format(FMT_STRING(":prompt sql ; '{}'"),
                                              shlex::escape(prql));
                            break;
                        }
                        case chart_type_t::spectro:
                            cmd = fmt::format(FMT_STRING(":spectrogram {}"),
                                              meta.lvm_name);
                            break;
                    }
                    if (!cmd.empty()) {
                        this->lss_exec_context
                            ->with_provenance(exec_context::mouse_input{})
                            ->execute(INTERNAL_SRC_LOC, cmd);
                    }
                }
                return true;
            }
            return false;
        }
        case 'h':
        case 'H':
        case NCKEY_LEFT:
            if (lv.get_left() == 0) {
                this->increase_line_context();
                lv.set_needs_update();
                return true;
            }
            break;
        case 'l':
        case 'L':
        case NCKEY_RIGHT:
            if (this->decrease_line_context()) {
                lv.set_needs_update();
                return true;
            }
            break;
    }
    return false;
}

std::optional<
    std::pair<grep_proc_source<vis_line_t>*, grep_proc_sink<vis_line_t>*>>
logfile_sub_source::get_grepper()
{
    return std::make_pair(
        (grep_proc_source<vis_line_t>*) &this->lss_meta_grepper,
        (grep_proc_sink<vis_line_t>*) &this->lss_meta_grepper);
}

/**
 * Functor for comparing the ld_file field of the logfile_data struct.
 */
struct logfile_data_eq {
    explicit logfile_data_eq(std::shared_ptr<logfile> lf)
        : lde_file(std::move(lf))
    {
    }

    bool operator()(
        const std::unique_ptr<logfile_sub_source::logfile_data>& ld) const
    {
        return this->lde_file == ld->get_file();
    }

    std::shared_ptr<logfile> lde_file;
};

bool
logfile_sub_source::insert_file(const std::shared_ptr<logfile>& lf)
{
    iterator existing;

    require_lt(lf->size(), MAX_LINES_PER_FILE);

    existing = std::find_if(this->lss_files.begin(),
                            this->lss_files.end(),
                            logfile_data_eq(nullptr));
    if (existing == this->lss_files.end()) {
        if (this->lss_files.size() >= MAX_FILES) {
            return false;
        }

        auto ld = std::make_unique<logfile_data>(
            this->lss_files.size(), this->get_filters(), lf);
        ld->set_visibility(lf->get_open_options().loo_is_visible);
        this->lss_files.push_back(std::move(ld));
    } else {
        (*existing)->set_file(lf);
    }

    return true;
}

Result<void, lnav::console::user_message>
logfile_sub_source::set_sql_filter(std::string stmt_str, sqlite3_stmt* stmt)
{
    if (stmt != nullptr && !this->lss_filtered_index.empty()) {
        auto top_cl = this->at(0_vl);
        auto ld = this->find_data(top_cl);
        auto eval_res
            = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());

        if (eval_res.isErr()) {
            sqlite3_finalize(stmt);
            return Err(eval_res.unwrapErr());
        }
    }

    for (auto& ld : *this) {
        ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
    }

    auto old_filter_iter = this->tss_filters.find(0);
    if (stmt != nullptr) {
        auto new_filter
            = std::make_shared<sql_filter>(*this, std::move(stmt_str), stmt);

        if (old_filter_iter != this->tss_filters.end()) {
            *old_filter_iter = new_filter;
        } else {
            this->tss_filters.add_filter(new_filter);
        }
    } else if (old_filter_iter != this->tss_filters.end()) {
        this->tss_filters.delete_filter((*old_filter_iter)->get_id());
    }

    return Ok();
}

Result<void, lnav::console::user_message>
logfile_sub_source::set_sql_marker(std::string stmt_str, sqlite3_stmt* stmt)
{
    static auto op = lnav_operation{"set_sql_marker"};
    if (stmt != nullptr && !this->lss_filtered_index.empty()) {
        auto top_cl = this->at(0_vl);
        auto ld = this->find_data(top_cl);
        auto eval_res
            = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());

        if (eval_res.isErr()) {
            sqlite3_finalize(stmt);
            return Err(eval_res.unwrapErr());
        }
    }

    auto op_guard = lnav_opid_guard::internal(op);
    log_info("setting SQL marker: %s", stmt_str.c_str());
    this->lss_marker_stmt_text = std::move(stmt_str);
    this->lss_marker_stmt = stmt;

    if (this->tss_view == nullptr || this->lss_force_rebuild) {
        log_info("skipping SQL marker update");
        return Ok();
    }

    auto& vis_bm = this->tss_view->get_bookmarks();
    auto& expr_marks_bv = vis_bm[&textview_curses::BM_USER_EXPR];
    auto& cl_marks_bv = this->lss_user_marks[&textview_curses::BM_USER_EXPR];

    expr_marks_bv.clear();
    if (this->lss_index_delegate) {
        this->lss_index_delegate->index_start(*this);
    }
    for (auto row = 0_vl; row < vis_line_t(this->lss_filtered_index.size());
         row += 1_vl)
    {
        auto cl = this->at(row);
        uint64_t line_number;
        auto ld = this->find_data(cl, line_number);

        if (!(*ld)->is_visible()) {
            continue;
        }
        auto ll = (*ld)->get_file()->begin() + line_number;
        if (ll->is_continued() || ll->is_ignored()) {
            continue;
        }
        auto eval_res
            = this->eval_sql_filter(this->lss_marker_stmt.in(), ld, ll);

        if (eval_res.isErr()) {
            ll->set_expr_mark(false);
        } else {
            auto matched = eval_res.unwrap();

            ll->set_expr_mark(matched);
            if (matched) {
                expr_marks_bv.insert_once(row);
                cl_marks_bv.insert_once(cl);
            }
        }
        if (this->lss_index_delegate) {
            this->lss_index_delegate->index_line(
                *this, (*ld)->get_file_ptr(), ll);
        }
    }
    if (this->lss_index_delegate) {
        this->lss_index_delegate->index_complete(*this);
    }
    log_info("SQL marker update complete");

    return Ok();
}

Result<void, lnav::console::user_message>
logfile_sub_source::set_preview_sql_filter(sqlite3_stmt* stmt)
{
    if (stmt != nullptr && !this->lss_filtered_index.empty()) {
        auto top_cl = this->at(0_vl);
        auto ld = this->find_data(top_cl);
        auto eval_res
            = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());

        if (eval_res.isErr()) {
            sqlite3_finalize(stmt);
            return Err(eval_res.unwrapErr());
        }
    }

    this->lss_preview_filter_stmt = stmt;

    return Ok();
}

Result<bool, lnav::console::user_message>
logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
                                    iterator ld,
                                    logfile::const_iterator ll)
{
    if (stmt == nullptr) {
        return Ok(false);
    }

    auto* lf = (*ld)->get_file_ptr();
    char timestamp_buffer[64];
    shared_buffer_ref raw_sbr;
    logline_value_vector values;
    auto& sbr = values.lvv_sbr;
    lf->read_full_message(ll, sbr);
    sbr.erase_ansi();
    auto format = lf->get_format();
    string_attrs_t sa;
    auto line_number = std::distance(lf->cbegin(), ll);
    format->annotate(lf, line_number, sa, values);
    auto lffs = lf->get_format_file_state();

    sqlite3_reset(stmt);
    sqlite3_clear_bindings(stmt);

    auto count = sqlite3_bind_parameter_count(stmt);
    for (int lpc = 0; lpc < count; lpc++) {
        const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);

        if (name[0] == '$') {
            const char* env_value;

            if ((env_value = getenv(&name[1])) != nullptr) {
                sqlite3_bind_text(stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
            }
            continue;
        }
        if (strcmp(name, ":log_level") == 0) {
            auto lvl = ll->get_level_name();
            sqlite3_bind_text(
                stmt, lpc + 1, lvl.data(), lvl.length(), SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_time") == 0) {
            auto len = sql_strftime(timestamp_buffer,
                                    sizeof(timestamp_buffer),
                                    ll->get_timeval(),
                                    'T');
            sqlite3_bind_text(
                stmt, lpc + 1, timestamp_buffer, len, SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_time_msecs") == 0) {
            sqlite3_bind_int64(
                stmt,
                lpc + 1,
                ll->get_time<std::chrono::milliseconds>().count());
            continue;
        }
        if (strcmp(name, ":log_mark") == 0) {
            sqlite3_bind_int(stmt, lpc + 1, ll->is_marked());
            continue;
        }
        if (strcmp(name, ":log_comment") == 0) {
            const auto& bm = lf->get_bookmark_metadata();
            auto line_number
                = static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
            auto bm_iter = bm.find(line_number);
            if (bm_iter != bm.end() && !bm_iter->second.bm_comment.empty()) {
                const auto& meta = bm_iter->second;
                sqlite3_bind_text(stmt,
                                  lpc + 1,
                                  meta.bm_comment.c_str(),
                                  meta.bm_comment.length(),
                                  SQLITE_STATIC);
            }
            continue;
        }
        if (strcmp(name, ":log_annotations") == 0) {
            const auto& bm = lf->get_bookmark_metadata();
            auto line_number
                = static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
            auto bm_iter = bm.find(line_number);
            if (bm_iter != bm.end()
                && !bm_iter->second.bm_annotations.la_pairs.empty())
            {
                const auto& meta = bm_iter->second;
                auto anno_str = logmsg_annotations_handlers.to_string(
                    meta.bm_annotations);

                sqlite3_bind_text(stmt,
                                  lpc + 1,
                                  anno_str.c_str(),
                                  anno_str.length(),
                                  SQLITE_TRANSIENT);
            }
            continue;
        }
        if (strcmp(name, ":log_tags") == 0) {
            const auto& bm = lf->get_bookmark_metadata();
            auto line_number
                = static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
            auto bm_iter = bm.find(line_number);
            if (bm_iter != bm.end() && !bm_iter->second.bm_tags.empty()) {
                const auto& meta = bm_iter->second;
                yajlpp_gen gen;

                yajl_gen_config(gen, yajl_gen_beautify, false);

                {
                    yajlpp_array arr(gen);

                    for (const auto& entry : meta.bm_tags) {
                        arr.gen(entry.te_tag);
                    }
                }

                string_fragment sf = gen.to_string_fragment();

                sqlite3_bind_text(
                    stmt, lpc + 1, sf.data(), sf.length(), SQLITE_TRANSIENT);
            }
            continue;
        }
        if (strcmp(name, ":log_format") == 0) {
            const auto format_name = format->get_name();
            sqlite3_bind_text(stmt,
                              lpc + 1,
                              format_name.get(),
                              format_name.size(),
                              SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_format_regex") == 0) {
            const auto pat_name = format->get_pattern_name(
                lffs.lffs_pattern_locks, line_number);
            sqlite3_bind_text(
                stmt, lpc + 1, pat_name.get(), pat_name.size(), SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_path") == 0) {
            const auto& filename = lf->get_filename();
            sqlite3_bind_text(stmt,
                              lpc + 1,
                              filename.c_str(),
                              filename.native().length(),
                              SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_unique_path") == 0) {
            const auto& filename = lf->get_unique_path();
            sqlite3_bind_text(stmt,
                              lpc + 1,
                              filename.c_str(),
                              filename.native().length(),
                              SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_text") == 0) {
            sqlite3_bind_text(
                stmt, lpc + 1, sbr.get_data(), sbr.length(), SQLITE_STATIC);
            continue;
        }
        if (strcmp(name, ":log_body") == 0) {
            auto body_attr_opt = get_string_attr(sa, SA_BODY);
            if (body_attr_opt) {
                const auto& sar
                    = body_attr_opt.value().saw_string_attr->sa_range;

                sqlite3_bind_text(stmt,
                                  lpc + 1,
                                  sbr.get_data_at(sar.lr_start),
                                  sar.length(),
                                  SQLITE_STATIC);
            } else {
                sqlite3_bind_null(stmt, lpc + 1);
            }
            continue;
        }
        if (strcmp(name, ":log_raw_text") == 0) {
            auto res = lf->read_raw_message(ll);

            if (res.isOk()) {
                raw_sbr = res.unwrap();
                sqlite3_bind_text(stmt,
                                  lpc + 1,
                                  raw_sbr.get_data(),
                                  raw_sbr.length(),
                                  SQLITE_STATIC);
            }
            continue;
        }
        if (strcmp(name, ":log_opid") == 0) {
            bind_to_sqlite(stmt, lpc + 1, values.lvv_opid_value);
            continue;
        }
        if (strcmp(name, ":log_opid_definition") == 0) {
            if (values.lvv_opid_value) {
                auto opids = lf->get_opids().readAccess();

                auto iter = opids->los_opid_ranges.find(
                    values.lvv_opid_value.value());
                if (iter != opids->los_opid_ranges.end()
                    && iter->second.otr_description.lod_index)
                {
                    const auto& opid_def
                        = (*format->lf_opid_description_def_vec)
                            [iter->second.otr_description.lod_index.value()];
                    bind_to_sqlite(stmt, lpc + 1, opid_def->od_name);
                } else {
                    sqlite3_bind_null(stmt, lpc + 1);
                }
            } else {
                sqlite3_bind_null(stmt, lpc + 1);
            }
            continue;
        }
        if (strcmp(name, ":log_src_file") == 0) {
            bind_to_sqlite(stmt, lpc + 1, values.lvv_src_file_value);
            continue;
        }
        if (strcmp(name, ":log_src_line") == 0) {
            bind_to_sqlite(stmt, lpc + 1, values.lvv_src_line_value);
            continue;
        }
        if (strcmp(name, ":log_thread_id") == 0) {
            bind_to_sqlite(stmt, lpc + 1, values.lvv_thread_id_value);
            continue;
        }
        for (const auto& lv : values.lvv_values) {
            if (lv.lv_meta.lvm_name != &name[1]) {
                continue;
            }

            switch (lv.lv_meta.lvm_kind) {
                case value_kind_t::VALUE_BOOLEAN:
                    sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
                    break;
                case value_kind_t::VALUE_FLOAT:
                    sqlite3_bind_double(stmt, lpc + 1, lv.lv_value.d);
                    break;
                case value_kind_t::VALUE_INTEGER:
                    sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
                    break;
                case value_kind_t::VALUE_NULL:
                    sqlite3_bind_null(stmt, lpc + 1);
                    break;
                default:
                    sqlite3_bind_text(stmt,
                                      lpc + 1,
                                      lv.text_value(),
                                      lv.text_length(),
                                      SQLITE_TRANSIENT);
                    break;
            }
            break;
        }
    }

    auto step_res = sqlite3_step(stmt);

    sqlite3_reset(stmt);
    sqlite3_clear_bindings(stmt);
    switch (step_res) {
        case SQLITE_OK:
        case SQLITE_DONE:
            return Ok(false);
        case SQLITE_ROW:
            return Ok(true);
        default:
            return Err(sqlite3_error_to_user_message(sqlite3_db_handle(stmt)));
    }
}

bool
logfile_sub_source::check_extra_filters(iterator ld, logfile::iterator ll)
{
    auto retval = true;

    if (this->lss_marked_only) {
        auto found_mark = ll->is_marked() || ll->is_expr_marked();
        auto to_start_ll = ll;
        while (!found_mark && to_start_ll->is_continued()) {
            if (to_start_ll->is_marked() || to_start_ll->is_expr_marked()) {
                found_mark = true;
            }
            --to_start_ll;
        }
        auto to_end_ll = std::next(ll);
        while (!found_mark && to_end_ll != (*ld)->get_file_ptr()->end()
               && to_end_ll->is_continued())
        {
            if (to_end_ll->is_marked() || to_end_ll->is_expr_marked()) {
                found_mark = true;
            }
            ++to_end_ll;
        }
        if (!found_mark) {
            retval = false;
        }
    }

    if (ll->get_msg_level() < this->tss_min_log_level) {
        this->tss_level_filtered_count += 1;
        retval = false;
    }

    if (*ll < this->ttt_min_row_time) {
        retval = false;
    }

    if (!(*ll <= this->ttt_max_row_time)) {
        retval = false;
    }

    return retval;
}

void
logfile_sub_source::invalidate_sql_filter()
{
    for (auto& ld : *this) {
        ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
    }
}

void
logfile_sub_source::text_mark(const bookmark_type_t* bm,
                              vis_line_t line,
                              bool added)
{
    if (line >= (int) this->lss_index.size()) {
        return;
    }

    auto cl = this->at(line);

    if (bm == &textview_curses::BM_USER) {
        auto* ll = this->find_line(cl);

        ll->set_mark(added);
    }
    if (added) {
        this->lss_user_marks[bm].insert_once(cl);
    } else {
        this->lss_user_marks[bm].erase(cl);
    }
    if (bm == &textview_curses::BM_META
        && this->lss_meta_grepper.gps_proc != nullptr)
    {
        this->tss_view->search_range(
            line, grep_proc<vis_line_t>::until_line(line + 1_vl));
        this->tss_view->search_new_data();
    }
}

void
logfile_sub_source::text_clear_marks(const bookmark_type_t* bm)
{
    if (bm == &textview_curses::BM_USER) {
        for (auto iter = this->lss_user_marks[bm].bv_tree.begin();
             iter != this->lss_user_marks[bm].bv_tree.end();
             ++iter)
        {
            this->find_line(*iter)->set_mark(false);
        }
    }
    this->lss_user_marks[bm].clear();
}

void
logfile_sub_source::remove_file(std::shared_ptr<logfile> lf)
{
    auto iter = std::find_if(
        this->lss_files.begin(), this->lss_files.end(), logfile_data_eq(lf));
    if (iter != this->lss_files.end()) {
        int file_index = iter - this->lss_files.begin();

        (*iter)->clear();
        for (auto& bv : this->lss_user_marks) {
            auto mark_curr = content_line_t(file_index * MAX_LINES_PER_FILE);
            auto mark_end
                = content_line_t((file_index + 1) * MAX_LINES_PER_FILE);
            auto file_range = bv.equal_range(mark_curr, mark_end);

            if (file_range.first != file_range.second) {
                auto to_del = std::vector<content_line_t>{};
                for (auto file_iter = file_range.first;
                     file_iter != file_range.second;
                     ++file_iter)
                {
                    to_del.emplace_back(*file_iter);
                }

                for (auto cl : to_del) {
                    bv.erase(cl);
                }
            }
        }

        this->lss_force_rebuild = true;
    }
    while (!this->lss_files.empty()) {
        if (this->lss_files.back()->get_file_ptr() == nullptr) {
            this->lss_files.pop_back();
        } else {
            break;
        }
    }
    this->lss_token_file = nullptr;
}

std::optional<vis_line_t>
logfile_sub_source::find_from_content(content_line_t cl)
{
    content_line_t line = cl;
    std::shared_ptr<logfile> lf = this->find(line);

    if (lf != nullptr) {
        auto ll_iter = lf->begin() + line;
        auto& ll = *ll_iter;
        auto vis_start_opt = this->find_from_time(ll.get_timeval());

        if (!vis_start_opt) {
            return std::nullopt;
        }

        auto vis_start = *vis_start_opt;

        while (vis_start < vis_line_t(this->text_line_count())) {
            content_line_t guess_cl = this->at(vis_start);

            if (cl == guess_cl) {
                return vis_start;
            }

            auto guess_line = this->find_line(guess_cl);

            if (!guess_line || ll < *guess_line) {
                return std::nullopt;
            }

            ++vis_start;
        }
    }

    return std::nullopt;
}

void
logfile_sub_source::reload_index_delegate()
{
    if (this->lss_index_delegate == nullptr) {
        return;
    }

    this->lss_index_delegate->index_start(*this);
    for (const auto index : this->lss_filtered_index) {
        auto cl = this->lss_index[index].value();
        uint64_t line_number;
        auto ld = this->find_data(cl, line_number);
        std::shared_ptr<logfile> lf = (*ld)->get_file();

        this->lss_index_delegate->index_line(
            *this, lf.get(), lf->begin() + line_number);
    }
    this->lss_index_delegate->index_complete(*this);
}

std::optional<std::shared_ptr<text_filter>>
logfile_sub_source::get_sql_filter()
{
    return this->tss_filters | lnav::itertools::find_if([](const auto& filt) {
               return filt->get_index() == 0
                   && dynamic_cast<sql_filter*>(filt.get()) != nullptr;
           })
        | lnav::itertools::deref();
}

void
log_location_history::loc_history_append(vis_line_t top)
{
    if (top < 0_vl || top >= vis_line_t(this->llh_log_source.text_line_count()))
    {
        return;
    }

    auto cl = this->llh_log_source.at(top);

    auto iter = this->llh_history.begin();
    iter += this->llh_history.size() - this->lh_history_position;
    this->llh_history.erase_from(iter);
    this->lh_history_position = 0;
    this->llh_history.push_back(cl);
}

std::optional<vis_line_t>
log_location_history::loc_history_back(vis_line_t current_top)
{
    while (this->lh_history_position < this->llh_history.size()) {
        auto iter = this->llh_history.rbegin();

        auto vis_for_pos = this->llh_log_source.find_from_content(*iter);

        if (this->lh_history_position == 0 && vis_for_pos != current_top) {
            return vis_for_pos;
        }

        if ((this->lh_history_position + 1) >= this->llh_history.size()) {
            break;
        }

        this->lh_history_position += 1;

        iter += this->lh_history_position;

        vis_for_pos = this->llh_log_source.find_from_content(*iter);

        if (vis_for_pos) {
            return vis_for_pos;
        }
    }

    return std::nullopt;
}

std::optional<vis_line_t>
log_location_history::loc_history_forward(vis_line_t current_top)
{
    while (this->lh_history_position > 0) {
        this->lh_history_position -= 1;

        auto iter = this->llh_history.rbegin();

        iter += this->lh_history_position;

        auto vis_for_pos = this->llh_log_source.find_from_content(*iter);

        if (vis_for_pos) {
            return vis_for_pos;
        }
    }

    return std::nullopt;
}

bool
sql_filter::matches(std::optional<line_source> ls_opt,
                    const shared_buffer_ref& line)
{
    if (!ls_opt) {
        return false;
    }

    auto ls = ls_opt;

    if (!ls->ls_line->is_message()) {
        return false;
    }
    if (this->sf_filter_stmt == nullptr) {
        return false;
    }

    auto lfp = ls->ls_file.shared_from_this();
    auto ld = this->sf_log_source.find_data_i(lfp);
    if (ld == this->sf_log_source.end()) {
        return false;
    }

    auto eval_res = this->sf_log_source.eval_sql_filter(
        this->sf_filter_stmt, ld, ls->ls_line);
    if (eval_res.unwrapOr(true)) {
        return false;
    }

    return true;
}

std::string
sql_filter::to_command() const
{
    return fmt::format(FMT_STRING("filter-expr {}"), this->lf_id);
}

std::optional<line_info>
logfile_sub_source::meta_grepper::grep_value_for_line(vis_line_t line,
                                                      std::string& value_out)
{
    auto line_meta_opt = this->lmg_source.find_bookmark_metadata(line);
    if (!line_meta_opt) {
        value_out.clear();
    } else {
        auto& bm = *(line_meta_opt.value());

        {
            md2attr_line mdal;

            auto parse_res = md4cpp::parse(bm.bm_comment, mdal);
            if (parse_res.isOk()) {
                value_out.append(parse_res.unwrap().get_string());
            } else {
                value_out.append(bm.bm_comment);
            }
        }

        value_out.append("\x1c");
        for (const auto& entry : bm.bm_tags) {
            value_out.append(entry.te_tag);
            value_out.append("\x1c");
        }
        value_out.append("\x1c");
        for (const auto& pair : bm.bm_annotations.la_pairs) {
            value_out.append(pair.first);
            value_out.append("\x1c");

            md2attr_line mdal;

            auto parse_res = md4cpp::parse(pair.second, mdal);
            if (parse_res.isOk()) {
                value_out.append(parse_res.unwrap().get_string());
            } else {
                value_out.append(pair.second);
            }
            value_out.append("\x1c");
        }
        value_out.append("\x1c");
        value_out.append(bm.bm_opid);
    }

    if (!this->lmg_done) {
        return line_info{};
    }

    return std::nullopt;
}

vis_line_t
logfile_sub_source::meta_grepper::grep_initial_line(vis_line_t start,
                                                    vis_line_t highest)
{
    auto& bm = this->lmg_source.tss_view->get_bookmarks();
    auto& bv = bm[&textview_curses::BM_META];

    if (bv.empty()) {
        return -1_vl;
    }
    return *bv.bv_tree.begin();
}

void
logfile_sub_source::meta_grepper::grep_next_line(vis_line_t& line)
{
    auto& bm = this->lmg_source.tss_view->get_bookmarks();
    auto& bv = bm[&textview_curses::BM_META];

    auto line_opt = bv.next(vis_line_t(line));
    if (!line_opt) {
        this->lmg_done = true;
    }
    line = line_opt.value_or(-1_vl);
}

void
logfile_sub_source::meta_grepper::grep_begin(grep_proc<vis_line_t>& gp,
                                             vis_line_t start,
                                             vis_line_t stop)
{
    this->lmg_source.quiesce();

    this->lmg_source.tss_view->grep_begin(gp, start, stop);
}

void
logfile_sub_source::meta_grepper::grep_end(grep_proc<vis_line_t>& gp)
{
    this->lmg_source.tss_view->grep_end(gp);
}

void
logfile_sub_source::meta_grepper::grep_match(grep_proc<vis_line_t>& gp,
                                             vis_line_t line)
{
    this->lmg_source.tss_view->grep_match(gp, line);
}

static std::vector<breadcrumb::possibility>
timestamp_poss()
{
    const static std::vector<breadcrumb::possibility> retval = {
        breadcrumb::possibility{"-1 day"},
        breadcrumb::possibility{"-1h"},
        breadcrumb::possibility{"-30m"},
        breadcrumb::possibility{"-15m"},
        breadcrumb::possibility{"-5m"},
        breadcrumb::possibility{"-1m"},
        breadcrumb::possibility{"+1m"},
        breadcrumb::possibility{"+5m"},
        breadcrumb::possibility{"+15m"},
        breadcrumb::possibility{"+30m"},
        breadcrumb::possibility{"+1h"},
        breadcrumb::possibility{"+1 day"},
    };

    return retval;
}

static attr_line_t
to_display(const std::shared_ptr<logfile>& lf)
{
    const auto& loo = lf->get_open_options();
    attr_line_t retval;

    if (loo.loo_piper) {
        if (!lf->get_open_options().loo_piper->is_finished()) {
            retval.append("\u21bb "_list_glyph);
        }
    } else if (loo.loo_child_poller && loo.loo_child_poller->is_alive()) {
        retval.append("\u21bb "_list_glyph);
    }
    retval.append(lf->get_unique_path());

    return retval;
}

void
logfile_sub_source::text_crumbs_for_line(int line,
                                         std::vector<breadcrumb::crumb>& crumbs)
{
    text_sub_source::text_crumbs_for_line(line, crumbs);

    if (this->lss_filtered_index.empty()) {
        return;
    }

    auto vl = vis_line_t(line);
    auto bmc = this->get_bookmark_metadata_context(
        vl, bookmark_metadata::categories::partition);
    if (bmc.bmc_current_metadata) {
        const auto& name = bmc.bmc_current_metadata.value()->bm_name;
        auto key = text_anchors::to_anchor_string(name);
        auto display = attr_line_t()
                           .append("\u2291 "_symbol)
                           .append(lnav::roles::variable(name))
                           .move();
        crumbs.emplace_back(
            key,
            display,
            [this]() -> std::vector<breadcrumb::possibility> {
                auto& vb = this->tss_view->get_bookmarks();
                const auto& bv = vb[&textview_curses::BM_PARTITION];
                std::vector<breadcrumb::possibility> retval;

                for (const auto& vl : bv.bv_tree) {
                    auto meta_opt = this->find_bookmark_metadata(vl);
                    if (!meta_opt || meta_opt.value()->bm_name.empty()) {
                        continue;
                    }

                    const auto& name = meta_opt.value()->bm_name;
                    retval.emplace_back(text_anchors::to_anchor_string(name),
                                        name);
                }

                return retval;
            },
            [ec = this->lss_exec_context](const auto& part) {
                auto cmd = fmt::format(FMT_STRING(":goto {}"),
                                       part.template get<std::string>());
                ec->execute(INTERNAL_SRC_LOC, cmd);
            });
    }

    auto line_pair_opt = this->find_line_with_file(vl);
    if (!line_pair_opt) {
        return;
    }
    auto line_pair = line_pair_opt.value();
    auto& lf = line_pair.first;
    auto format = lf->get_format();
    char ts[64];

    sql_strftime(ts, sizeof(ts), line_pair.second->get_timeval(), 'T');

    crumbs.emplace_back(std::string(ts),
                        timestamp_poss,
                        [ec = this->lss_exec_context](const auto& ts) {
                            auto cmd
                                = fmt::format(FMT_STRING(":goto {}"),
                                              ts.template get<std::string>());
                            ec->execute(INTERNAL_SRC_LOC, cmd);
                        });
    crumbs.back().c_expected_input
        = breadcrumb::crumb::expected_input_t::anything;
    crumbs.back().c_search_placeholder = "(Enter an absolute or relative time)";

    auto format_name = format->get_name().to_string();

    crumbs.emplace_back(
        format_name,
        attr_line_t().append(format_name),
        [this]() -> std::vector<breadcrumb::possibility> {
            return this->lss_files
                | lnav::itertools::filter_in([](const auto& file_data) {
                       return file_data->is_visible();
                   })
                | lnav::itertools::map(&logfile_data::get_file_ptr)
                | lnav::itertools::map(&logfile::get_format_name)
                | lnav::itertools::unique()
                | lnav::itertools::map([](const auto& elem) {
                       return breadcrumb::possibility{
                           elem.to_string(),
                       };
                   })
                | lnav::itertools::to_vector();
        },
        [ec = this->lss_exec_context](const auto& format_name) {
            static const std::string MOVE_STMT = R"(;UPDATE lnav_views
     SET selection = ifnull(
         (SELECT log_line FROM all_logs WHERE log_format = $format_name LIMIT 1),
         (SELECT raise_error(
            'Could not find format: ' || $format_name,
            'The corresponding log messages might have been filtered out'))
       )
     WHERE name = 'log'
)";

            ec->execute_with(
                INTERNAL_SRC_LOC,
                MOVE_STMT,
                std::make_pair("format_name",
                               format_name.template get<std::string>()));
        });

    auto msg_start_iter = lf->message_start(line_pair.second);
    auto file_line_number = std::distance(lf->begin(), msg_start_iter);
    crumbs.emplace_back(
        lf->get_unique_path(),
        to_display(lf).appendf(FMT_STRING("[{:L}]"), file_line_number),
        [this]() -> std::vector<breadcrumb::possibility> {
            return this->lss_files
                | lnav::itertools::filter_in([](const auto& file_data) {
                       return file_data->is_visible();
                   })
                | lnav::itertools::map([](const auto& file_data) {
                       return breadcrumb::possibility{
                           file_data->get_file_ptr()->get_unique_path(),
                           to_display(file_data->get_file()),
                       };
                   });
        },
        [ec = this->lss_exec_context](const auto& uniq_path) {
            static const std::string MOVE_STMT = R"(;UPDATE lnav_views
     SET selection = ifnull(
          (SELECT log_line FROM all_logs WHERE log_unique_path = $uniq_path LIMIT 1),
          (SELECT raise_error(
            'Could not find file: ' || $uniq_path,
            'The corresponding log messages might have been filtered out'))
         )
     WHERE name = 'log'
)";

            ec->execute_with(
                INTERNAL_SRC_LOC,
                MOVE_STMT,
                std::make_pair("uniq_path",
                               uniq_path.template get<std::string>()));
        });

    shared_buffer sb;
    logline_value_vector values;
    auto& sbr = values.lvv_sbr;

    lf->read_full_message(msg_start_iter, sbr);
    attr_line_t al(to_string(sbr));
    if (!sbr.get_metadata().m_valid_utf) {
        scrub_to_utf8(al.al_string.data(), al.al_string.length());
    }
    if (sbr.get_metadata().m_has_ansi) {
        // bleh
        scrub_ansi_string(al.get_string(), &al.al_attrs);
        sbr.share(sb, al.al_string.data(), al.al_string.size());
    }
    format->annotate(lf.get(), file_line_number, al.get_attrs(), values);

    {
        static const std::string MOVE_STMT = R"(;UPDATE lnav_views
          SET selection = ifnull(
            (SELECT log_line FROM all_logs WHERE log_thread_id = $tid LIMIT 1),
            (SELECT raise_error('Could not find thread ID: ' || $tid,
                                'The corresponding log messages might have been filtered out')))
          WHERE name = 'log'
        )";
        static const auto ELLIPSIS = "\u22ef"_frag;

        auto tid_display = values.lvv_thread_id_value.has_value()
            ? lnav::roles::identifier(values.lvv_thread_id_value.value())
            : lnav::roles::hidden(ELLIPSIS);
        crumbs.emplace_back(
            (values.lvv_thread_id_value.has_value()
                 ? values.lvv_thread_id_value.value()
                 : ""_frag)
                .to_string(),
            attr_line_t()
                .append(ui_icon_t::thread)
                .append(" ")
                .append(tid_display),
            [this]() -> std::vector<breadcrumb::possibility> {
                std::set<std::string> poss_strs;

                for (const auto& file_data : this->lss_files) {
                    if (file_data->get_file_ptr() == nullptr) {
                        continue;
                    }
                    safe::ReadAccess<logfile::safe_thread_id_state> r_tid_map(
                        file_data->get_file_ptr()->get_thread_ids());

                    for (const auto& pair : r_tid_map->ltis_tid_ranges) {
                        poss_strs.emplace(pair.first.to_string());
                    }
                }

                std::vector<breadcrumb::possibility> retval;

                std::transform(poss_strs.begin(),
                               poss_strs.end(),
                               std::back_inserter(retval),
                               [](const auto& tid_str) {
                                   return breadcrumb::possibility(tid_str);
                               });

                return retval;
            },
            [ec = this->lss_exec_context](const auto& tid) {
                ec->execute_with(
                    INTERNAL_SRC_LOC,
                    MOVE_STMT,
                    std::make_pair("tid", tid.template get<std::string>()));
            });
    }

    {
        static const std::string MOVE_STMT = R"(;UPDATE lnav_views
          SET selection = ifnull(
            (SELECT log_line FROM all_logs WHERE log_opid = $opid LIMIT 1),
            (SELECT raise_error('Could not find opid: ' || $opid,
                                'The corresponding log messages might have been filtered out')))
          WHERE name = 'log'
        )";
        static const std::string ELLIPSIS = "\u22ef";

        auto opid_display = values.lvv_opid_value.has_value()
            ? lnav::roles::identifier(values.lvv_opid_value.value())
            : lnav::roles::hidden(ELLIPSIS);
        crumbs.emplace_back(
            values.lvv_opid_value.has_value() ? values.lvv_opid_value.value()
                                              : "",
            attr_line_t().append(opid_display),
            [this]() -> std::vector<breadcrumb::possibility> {
                std::unordered_set<std::string> poss_strs;

                for (const auto& file_data : this->lss_files) {
                    if (file_data->get_file_ptr() == nullptr) {
                        continue;
                    }
                    safe::ReadAccess<logfile::safe_opid_state> r_opid_map(
                        file_data->get_file_ptr()->get_opids());

                    poss_strs.reserve(poss_strs.size()
                                      + r_opid_map->los_opid_ranges.size());
                    for (const auto& pair : r_opid_map->los_opid_ranges) {
                        poss_strs.insert(pair.first.to_string());
                    }
                }

                std::vector<breadcrumb::possibility> retval;
                retval.reserve(poss_strs.size());

                std::transform(poss_strs.begin(),
                               poss_strs.end(),
                               std::back_inserter(retval),
                               [](const auto& opid_str) {
                                   return breadcrumb::possibility(opid_str);
                               });

                return retval;
            },
            [ec = this->lss_exec_context](const auto& opid) {
                ec->execute_with(
                    INTERNAL_SRC_LOC,
                    MOVE_STMT,
                    std::make_pair("opid", opid.template get<std::string>()));
            });
    }

    auto sf = string_fragment::from_str(al.get_string());
    auto body_opt = get_string_attr(al.get_attrs(), SA_BODY);
    auto nl_pos_opt = sf.find('\n');
    auto msg_line_number = std::distance(msg_start_iter, line_pair.second);
    auto line_from_top = line - msg_line_number;
    if (body_opt && nl_pos_opt) {
        if (this->lss_token_meta_line != file_line_number
            || this->lss_token_meta_size != sf.length())
        {
            if (body_opt->saw_string_attr->sa_range.length() < 128 * 1024) {
                this->lss_token_meta
                    = lnav::document::discover(al)
                          .over_range(
                              body_opt.value().saw_string_attr->sa_range)
                          .perform();
                // XXX discover_structure() changes `al`, have to recompute
                // stuff
                sf = al.to_string_fragment();
                body_opt = get_string_attr(al.get_attrs(), SA_BODY);
            } else {
                this->lss_token_meta = lnav::document::metadata{};
            }
            this->lss_token_meta_line = file_line_number;
            this->lss_token_meta_size = sf.length();
        }

        const auto initial_size = crumbs.size();
        auto sf_body
            = sf.sub_range(body_opt->saw_string_attr->sa_range.lr_start,
                           body_opt->saw_string_attr->sa_range.lr_end);
        file_off_t line_offset = body_opt->saw_string_attr->sa_range.lr_start;
        file_off_t line_end_offset = sf.length();
        ssize_t line_number = 0;

        for (const auto& sf_line : sf_body.split_lines()) {
            if (line_number >= msg_line_number) {
                line_end_offset = line_offset + sf_line.length();
                break;
            }
            line_number += 1;
            line_offset += sf_line.length();
        }

        this->lss_token_meta.m_sections_tree.visit_overlapping(
            line_offset,
            line_end_offset,
            [this,
             initial_size,
             meta = &this->lss_token_meta,
             &crumbs,
             line_from_top](const auto& iv) {
                auto path = crumbs | lnav::itertools::skip(initial_size)
                    | lnav::itertools::map(&breadcrumb::crumb::c_key)
                    | lnav::itertools::append(iv.value);
                auto curr_node = lnav::document::hier_node::lookup_path(
                    meta->m_sections_root.get(), path);

                crumbs.emplace_back(
                    iv.value,
                    [meta, path]() { return meta->possibility_provider(path); },
                    [this, curr_node, path, line_from_top](const auto& key) {
                        if (!curr_node) {
                            return;
                        }
                        auto* parent_node = curr_node.value()->hn_parent;
                        if (parent_node == nullptr) {
                            return;
                        }
                        key.match(
                            [parent_node](const std::string& str) {
                                return parent_node->find_line_number(str);
                            },
                            [parent_node](size_t index) {
                                return parent_node->find_line_number(index);
                            })
                            | [this, line_from_top](auto line_number) {
                                  this->tss_view->set_selection(
                                      vis_line_t(line_from_top + line_number));
                              };
                    });
                if (curr_node
                    && !curr_node.value()->hn_parent->is_named_only()) {
                    auto node = lnav::document::hier_node::lookup_path(
                        meta->m_sections_root.get(), path);

                    crumbs.back().c_expected_input
                        = curr_node.value()
                              ->hn_parent->hn_named_children.empty()
                        ? breadcrumb::crumb::expected_input_t::index
                        : breadcrumb::crumb::expected_input_t::index_or_exact;
                    crumbs.back().with_possible_range(
                        node | lnav::itertools::map([](const auto hn) {
                            return hn->hn_parent->hn_children.size();
                        })
                        | lnav::itertools::unwrap_or(size_t{0}));
                }
            });

        auto path = crumbs | lnav::itertools::skip(initial_size)
            | lnav::itertools::map(&breadcrumb::crumb::c_key);
        auto node = lnav::document::hier_node::lookup_path(
            this->lss_token_meta.m_sections_root.get(), path);

        if (node && !node.value()->hn_children.empty()) {
            auto poss_provider = [curr_node = node.value()]() {
                std::vector<breadcrumb::possibility> retval;
                for (const auto& child : curr_node->hn_named_children) {
                    retval.emplace_back(child.first);
                }
                return retval;
            };
            auto path_performer
                = [this, curr_node = node.value(), line_from_top](
                      const breadcrumb::crumb::key_t& value) {
                      value.match(
                          [curr_node](const std::string& str) {
                              return curr_node->find_line_number(str);
                          },
                          [curr_node](size_t index) {
                              return curr_node->find_line_number(index);
                          })
                          | [this, line_from_top](size_t line_number) {
                                this->tss_view->set_selection(
                                    vis_line_t(line_from_top + line_number));
                            };
                  };
            crumbs.emplace_back("", "\u22ef", poss_provider, path_performer);
            crumbs.back().c_expected_input
                = node.value()->hn_named_children.empty()
                ? breadcrumb::crumb::expected_input_t::index
                : breadcrumb::crumb::expected_input_t::index_or_exact;
        }
    }
}

void
logfile_sub_source::quiesce()
{
    for (auto& ld : this->lss_files) {
        auto* lf = ld->get_file_ptr();

        if (lf == nullptr) {
            continue;
        }

        lf->quiesce();
    }
}

bookmark_metadata&
logfile_sub_source::get_bookmark_metadata(content_line_t cl)
{
    auto line_pair = this->find_line_with_file(cl).value();
    auto line_number = static_cast<uint32_t>(
        std::distance(line_pair.first->begin(), line_pair.second));

    return line_pair.first->get_bookmark_metadata()[line_number];
}

logfile_sub_source::bookmark_metadata_context
logfile_sub_source::get_bookmark_metadata_context(
    vis_line_t vl, bookmark_metadata::categories desired) const
{
    const auto& vb = this->tss_view->get_bookmarks();
    const auto& bv = vb[desired == bookmark_metadata::categories::partition
                            ? &textview_curses::BM_PARTITION
                            : &textview_curses::BM_META];
    auto vl_iter = bv.bv_tree.lower_bound(vl + 1_vl);

    std::optional<vis_line_t> next_line;
    for (auto next_vl_iter = vl_iter; next_vl_iter != bv.bv_tree.end();
         ++next_vl_iter)
    {
        auto bm_opt = this->find_bookmark_metadata(*next_vl_iter);
        if (!bm_opt) {
            continue;
        }

        if (bm_opt.value()->has(desired)) {
            next_line = *next_vl_iter;
            break;
        }
    }
    if (vl_iter == bv.bv_tree.begin()) {
        return bookmark_metadata_context{std::nullopt, std::nullopt, next_line};
    }

    --vl_iter;
    while (true) {
        auto bm_opt = this->find_bookmark_metadata(*vl_iter);
        if (bm_opt) {
            if (bm_opt.value()->has(desired)) {
                return bookmark_metadata_context{
                    *vl_iter, bm_opt.value(), next_line};
            }
        }

        if (vl_iter == bv.bv_tree.begin()) {
            return bookmark_metadata_context{
                std::nullopt, std::nullopt, next_line};
        }
        --vl_iter;
    }
    return bookmark_metadata_context{std::nullopt, std::nullopt, next_line};
}

std::optional<bookmark_metadata*>
logfile_sub_source::find_bookmark_metadata(content_line_t cl) const
{
    auto line_pair = this->find_line_with_file(cl).value();
    auto line_number = static_cast<uint32_t>(
        std::distance(line_pair.first->begin(), line_pair.second));

    auto& bm = line_pair.first->get_bookmark_metadata();
    auto bm_iter = bm.find(line_number);
    if (bm_iter == bm.end()) {
        return std::nullopt;
    }

    return &bm_iter->second;
}

void
logfile_sub_source::erase_bookmark_metadata(content_line_t cl)
{
    auto line_pair = this->find_line_with_file(cl).value();
    auto line_number = static_cast<uint32_t>(
        std::distance(line_pair.first->begin(), line_pair.second));

    auto& bm = line_pair.first->get_bookmark_metadata();
    auto bm_iter = bm.find(line_number);
    if (bm_iter != bm.end()) {
        bm.erase(bm_iter);
    }
}

void
logfile_sub_source::clear_bookmark_metadata()
{
    for (auto& ld : *this) {
        if (ld->get_file_ptr() == nullptr) {
            continue;
        }

        ld->get_file_ptr()->get_bookmark_metadata().clear();
    }
}

void
logfile_sub_source::increase_line_context()
{
    auto old_context = this->lss_line_context;

    switch (this->lss_line_context) {
        case line_context_t::filename:
            // nothing to do
            break;
        case line_context_t::basename:
            this->lss_line_context = line_context_t::filename;
            break;
        case line_context_t::none:
            this->lss_line_context = line_context_t::basename;
            break;
        case line_context_t::time_column:
            this->lss_line_context = line_context_t::none;
            break;
    }
    if (old_context != this->lss_line_context) {
        this->clear_line_size_cache();
    }
}

bool
logfile_sub_source::decrease_line_context()
{
    static const auto& cfg
        = injector::get<const logfile_sub_source_ns::config&>();
    auto old_context = this->lss_line_context;

    switch (this->lss_line_context) {
        case line_context_t::filename:
            this->lss_line_context = line_context_t::basename;
            break;
        case line_context_t::basename:
            this->lss_line_context = line_context_t::none;
            break;
        case line_context_t::none:
            if (cfg.c_time_column
                != logfile_sub_source_ns::time_column_feature_t::Disabled)
            {
                this->lss_line_context = line_context_t::time_column;
            }
            break;
        case line_context_t::time_column:
            break;
    }
    if (old_context != this->lss_line_context) {
        this->clear_line_size_cache();

        return true;
    }

    return false;
}

size_t
logfile_sub_source::get_filename_offset() const
{
    switch (this->lss_line_context) {
        case line_context_t::filename:
            return this->lss_filename_width;
        case line_context_t::basename:
            return this->lss_basename_width;
        default:
            return 0;
    }
}

size_t
logfile_sub_source::file_count() const
{
    size_t retval = 0;
    const_iterator iter;

    for (iter = this->cbegin(); iter != this->cend(); ++iter) {
        if (*iter != nullptr && (*iter)->get_file() != nullptr) {
            retval += 1;
        }
    }

    return retval;
}

size_t
logfile_sub_source::text_size_for_line(textview_curses& tc,
                                       int row,
                                       text_sub_source::line_flags_t flags)
{
    size_t index = row % LINE_SIZE_CACHE_SIZE;

    if (this->lss_line_size_cache[index].first != row) {
        std::string value;

        this->text_value_for_line(tc, row, value, flags);
        scrub_ansi_string(value, nullptr);
        auto line_width = string_fragment::from_str(value).column_width();
        if (this->lss_line_context == line_context_t::time_column) {
            auto time_attr
                = find_string_attr(this->lss_token_al.al_attrs, &L_TIMESTAMP);
            if (time_attr != this->lss_token_al.al_attrs.end()) {
                line_width -= time_attr->sa_range.length();
                auto format = this->lss_token_file->get_format();
                if (format->lf_level_hideable) {
                    auto level_attr = find_string_attr(
                        this->lss_token_al.al_attrs, &L_LEVEL);
                    if (level_attr != this->lss_token_al.al_attrs.end()) {
                        line_width -= level_attr->sa_range.length();
                    }
                }
            }
        }
        this->lss_line_size_cache[index].second = line_width;
        this->lss_line_size_cache[index].first = row;
    }
    return this->lss_line_size_cache[index].second;
}

int
logfile_sub_source::get_filtered_count_for(size_t filter_index) const
{
    int retval = 0;

    for (const auto& ld : this->lss_files) {
        retval += ld->ld_filter_state.lfo_filter_state
                      .tfs_filter_hits[filter_index];
    }

    return retval;
}

std::optional<vis_line_t>
logfile_sub_source::row_for(const row_info& ri)
{
    auto lb = std::lower_bound(this->lss_filtered_index.begin(),
                               this->lss_filtered_index.end(),
                               ri.ri_time,
                               filtered_logline_cmp(*this));
    if (lb != this->lss_filtered_index.end()) {
        auto first_lb = lb;
        while (true) {
            auto cl = this->lss_index[*lb].value();
            if (content_line_t(ri.ri_id) == cl) {
                first_lb = lb;
                break;
            }
            auto ll = this->find_line(cl);
            if (ll->get_timeval() != ri.ri_time) {
                break;
            }
            auto next_lb = std::next(lb);
            if (next_lb == this->lss_filtered_index.end()) {
                break;
            }
            lb = next_lb;
        }

        const auto dst
            = std::distance(this->lss_filtered_index.begin(), first_lb);
        return vis_line_t(dst);
    }

    return std::nullopt;
}

std::unique_ptr<logline_window>
logfile_sub_source::window_at(vis_line_t start_vl, vis_line_t end_vl)
{
    return std::make_unique<logline_window>(*this, start_vl, end_vl);
}

std::unique_ptr<logline_window>
logfile_sub_source::window_at(vis_line_t start_vl)
{
    return std::make_unique<logline_window>(*this, start_vl, start_vl + 1_vl);
}

std::unique_ptr<logline_window>
logfile_sub_source::window_to_end(vis_line_t start_vl)
{
    return std::make_unique<logline_window>(
        *this, start_vl, vis_line_t(this->text_line_count()));
}

std::optional<vis_line_t>
logfile_sub_source::row_for_anchor(const std::string& id)
{
    if (startswith(id, "#msg")) {
        static const auto ANCHOR_RE
            = lnav::pcre2pp::code::from_const(R"(#msg([0-9a-fA-F]+)-(.+))");
        thread_local auto md = lnav::pcre2pp::match_data::unitialized();

        if (ANCHOR_RE.capture_from(id).into(md).found_p()) {
            auto scan_res = scn::scan<int64_t>(md[1]->to_string_view(), "{:x}");
            if (scan_res) {
                auto ts_low = std::chrono::microseconds{scan_res->value()};
                auto ts_high = ts_low + 1us;

                auto low_vl = this->row_for_time(to_timeval(ts_low));
                auto high_vl = this->row_for_time(to_timeval(ts_high));
                if (low_vl) {
                    auto lw = this->window_at(
                        low_vl.value(),
                        high_vl.value_or(low_vl.value() + 1_vl));

                    for (const auto& li : *lw) {
                        auto hash_res = li.get_line_hash();
                        if (hash_res.isErr()) {
                            auto errmsg = hash_res.unwrapErr();

                            log_error("unable to get line hash: %s",
                                      errmsg.c_str());
                            continue;
                        }

                        auto hash = hash_res.unwrap();
                        if (hash == md[2]) {
                            return li.get_vis_line();
                        }
                    }
                }
            }
        }

        return std::nullopt;
    }

    auto& vb = this->tss_view->get_bookmarks();
    const auto& bv = vb[&textview_curses::BM_PARTITION];

    for (const auto& vl : bv.bv_tree) {
        auto meta_opt = this->find_bookmark_metadata(vl);
        if (!meta_opt || meta_opt.value()->bm_name.empty()) {
            continue;
        }

        const auto& name = meta_opt.value()->bm_name;
        if (id == text_anchors::to_anchor_string(name)) {
            return vl;
        }
    }

    return std::nullopt;
}

std::optional<vis_line_t>
logfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
{
    if (vl < this->lss_filtered_index.size()) {
        auto file_and_line_pair = this->find_line_with_file(vl);
        if (file_and_line_pair) {
            const auto& [lf, line] = file_and_line_pair.value();
            if (line->is_continued()) {
                auto retval = vl;
                switch (dir) {
                    case direction::prev: {
                        auto first_line = line;
                        while (first_line->is_continued()) {
                            --first_line;
                            retval -= 1_vl;
                        }
                        return retval;
                    }
                    case direction::next: {
                        auto first_line = line;
                        while (first_line->is_continued()) {
                            ++first_line;
                            retval += 1_vl;
                        }
                        return retval;
                    }
                }
            }
        }
    }

    auto bmc = this->get_bookmark_metadata_context(
        vl, bookmark_metadata::categories::partition);
    switch (dir) {
        case direction::prev: {
            if (bmc.bmc_current && bmc.bmc_current.value() != vl) {
                return bmc.bmc_current;
            }
            if (!bmc.bmc_current || bmc.bmc_current.value() == 0_vl) {
                return 0_vl;
            }
            auto prev_bmc = this->get_bookmark_metadata_context(
                bmc.bmc_current.value() - 1_vl,
                bookmark_metadata::categories::partition);
            if (!prev_bmc.bmc_current) {
                return 0_vl;
            }
            return prev_bmc.bmc_current;
        }
        case direction::next:
            return bmc.bmc_next_line;
    }
    return std::nullopt;
}

std::optional<std::string>
logfile_sub_source::anchor_for_row(vis_line_t vl)
{
    auto line_meta = this->get_bookmark_metadata_context(
        vl, bookmark_metadata::categories::partition);
    if (!line_meta.bmc_current_metadata) {
        auto lw = window_at(vl);

        for (const auto& li : *lw) {
            auto hash_res = li.get_line_hash();
            if (hash_res.isErr()) {
                auto errmsg = hash_res.unwrapErr();
                log_error("unable to compute line hash: %s", errmsg.c_str());
                break;
            }
            auto hash = hash_res.unwrap();
            auto retval = fmt::format(
                FMT_STRING("#msg{:016x}-{}"),
                li.get_logline().get_time<std::chrono::microseconds>().count(),
                hash);

            return retval;
        }

        return std::nullopt;
    }

    return text_anchors::to_anchor_string(
        line_meta.bmc_current_metadata.value()->bm_name);
}

std::unordered_set<std::string>
logfile_sub_source::get_anchors()
{
    auto& vb = this->tss_view->get_bookmarks();
    const auto& bv = vb[&textview_curses::BM_PARTITION];
    std::unordered_set<std::string> retval;

    for (const auto& vl : bv.bv_tree) {
        auto meta_opt = this->find_bookmark_metadata(vl);
        if (!meta_opt || meta_opt.value()->bm_name.empty()) {
            continue;
        }

        const auto& name = meta_opt.value()->bm_name;
        retval.emplace(text_anchors::to_anchor_string(name));
    }

    return retval;
}

bool
logfile_sub_source::text_handle_mouse(
    textview_curses& tc,
    const listview_curses::display_line_content_t& mouse_line,
    mouse_event& me)
{
    if (mouse_line.is<listview_curses::static_overlay_content>()
        && this->text_line_count() > 0)
    {
        auto top = tc.get_top();
        if (top > 0) {
            auto win = this->window_at(top - 1_vl);
            for (const auto& li : *win) {
                tc.set_top(li.get_vis_line());
                tc.set_selection(li.get_vis_line());
                return true;
            }
        }
    }

    if (tc.get_overlay_selection()) {
        auto nci = ncinput{};
        if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 2, 4)) {
            nci.id = ' ';
            nci.eff_text[0] = ' ';
            this->list_input_handle_key(tc, nci);
        } else if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 5, 6)) {
            nci.id = '#';
            nci.eff_text[0] = '#';
            this->list_input_handle_key(tc, nci);
        }
    }
    return true;
}

void
logfile_sub_source::reload_config(error_reporter& reporter)
{
    static const auto& cfg
        = injector::get<const logfile_sub_source_ns::config&>();

    switch (cfg.c_time_column) {
        case logfile_sub_source_ns::time_column_feature_t::Default:
            if (this->lss_line_context == line_context_t::none) {
                this->lss_line_context = line_context_t::time_column;
            }
            break;
        case logfile_sub_source_ns::time_column_feature_t::Disabled:
            if (this->lss_line_context == line_context_t::time_column) {
                this->lss_line_context = line_context_t::none;
            }
            break;
        case logfile_sub_source_ns::time_column_feature_t::Enabled:
            break;
    }
}

void
logfile_sub_source::clear_preview()
{
    text_sub_source::clear_preview();

    this->set_preview_sql_filter(nullptr);
    auto last = std::remove_if(this->lss_highlighters.begin(),
                               this->lss_highlighters.end(),
                               [](const auto& hl) { return hl.h_preview; });
    this->lss_highlighters.erase(last, this->lss_highlighters.end());
}

void
logfile_sub_source::add_commands_for_session(
    const std::function<void(const std::string&)>& receiver)
{
    text_sub_source::add_commands_for_session(receiver);

    auto mark_expr = this->get_sql_marker_text();
    if (!mark_expr.empty()) {
        receiver(fmt::format(FMT_STRING("mark-expr {}"), mark_expr));
    }

    for (const auto& hl : this->lss_highlighters) {
        auto cmd = "highlight-field "s;
        if (hl.h_attrs.has_style(text_attrs::style::bold)) {
            cmd += "--bold ";
        }
        if (hl.h_attrs.has_style(text_attrs::style::underline)) {
            cmd += "--underline ";
        }
        if (hl.h_attrs.has_style(text_attrs::style::blink)) {
            cmd += "--blink ";
        }
        if (hl.h_attrs.has_style(text_attrs::style::struck)) {
            cmd += "--strike ";
        }
        if (hl.h_attrs.has_style(text_attrs::style::italic)) {
            cmd += "--italic ";
        }
        cmd += hl.h_field.to_string() + " " + hl.h_regex->get_pattern();
        receiver(cmd);
    }

    for (const auto& format : log_format::get_root_formats()) {
        auto field_states = format->get_field_states();

        for (const auto& fs_pair : field_states) {
            if (!fs_pair.second.lvm_user_hidden) {
                continue;
            }

            if (fs_pair.second.lvm_user_hidden.value()) {
                receiver(fmt::format(FMT_STRING("hide-fields {}.{}"),
                                     format->get_name().to_string(),
                                     fs_pair.first.to_string()));
            } else if (fs_pair.second.lvm_hidden) {
                receiver(fmt::format(FMT_STRING("show-fields {}.{}"),
                                     format->get_name().to_string(),
                                     fs_pair.first.to_string()));
            }
        }
    }

    for (const auto& [schema_id, bp] : this->lss_breakpoints) {
        if (bp.bp_source != breakpoint_info::source_type::src_location) {
            continue;
        }
        receiver(fmt::format(FMT_STRING("breakpoint {}"), bp.bp_description));
    }
}

void
logfile_sub_source::update_filter_hash_state(hasher& h) const
{
    text_sub_source::update_filter_hash_state(h);

    for (const auto& ld : this->lss_files) {
        if (ld->get_file_ptr() == nullptr || !ld->is_visible()) {
            h.update(0);
        } else {
            h.update(1);
        }
    }
    h.update(this->tss_min_log_level);
    h.update(this->lss_marked_only);
}

tstack / lnav / 23836075406-2909

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous