17589970077-2502

Committed 09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%

Build # 17589970077-2502

Build Type

push

github

Committed by

tstack

Commit Message

[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

Run Details

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.08

/src/log_format.cc

/**
 * Copyright (c) 2007-2015, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <algorithm>
#include <memory>

#include <fnmatch.h>
#include <stdio.h>
#include <string.h>

#include "base/fs_util.hh"
#include "base/is_utf8.hh"
#include "base/itertools.enumerate.hh"
#include "base/itertools.hh"
#include "base/map_util.hh"
#include "base/opt_util.hh"
#include "base/snippet_highlighters.hh"
#include "base/string_util.hh"
#include "command_executor.hh"
#include "config.h"
#include "fmt/format.h"
#include "lnav_util.hh"
#include "log_format_ext.hh"
#include "log_search_table.hh"
#include "log_vtab_impl.hh"
#include "ptimec.hh"
#include "readline_highlighters.hh"
#include "scn/scan.h"
#include "spookyhash/SpookyV2.h"
#include "sql_util.hh"
#include "sqlite-extension-func.hh"
#include "sqlitepp.hh"
#include "yajlpp/yajlpp.hh"
#include "yajlpp/yajlpp_def.hh"

using namespace lnav::roles::literals;

static auto intern_lifetime = intern_string::get_table_lifetime();

constexpr string_attr_type<void> L_PREFIX("prefix");
constexpr string_attr_type<void> L_TIMESTAMP("timestamp");
constexpr string_attr_type<void> L_LEVEL("level");
constexpr string_attr_type<std::shared_ptr<logfile>> L_FILE("file");
constexpr string_attr_type<bookmark_metadata*> L_PARTITION("partition");
constexpr string_attr_type<void> L_MODULE("module");
constexpr string_attr_type<void> L_OPID("opid");
constexpr string_attr_type<bookmark_metadata*> L_META("meta");

external_log_format::mod_map_t external_log_format::MODULE_FORMATS;
std::vector<std::shared_ptr<external_log_format>>
    external_log_format::GRAPH_ORDERED_FORMATS;

const intern_string_t log_format::LOG_TIME_STR
    = intern_string::lookup("log_time");
const intern_string_t log_format::LOG_LEVEL_STR
    = intern_string::lookup("log_level");
const intern_string_t log_format::LOG_OPID_STR
    = intern_string::lookup("log_opid");

static constexpr uint32_t DATE_TIME_SET_FLAGS = ETF_YEAR_SET | ETF_MONTH_SET
    | ETF_DAY_SET | ETF_HOUR_SET | ETF_MINUTE_SET | ETF_SECOND_SET;

log_level_stats&
log_level_stats::operator|=(const log_level_stats& rhs)
{
    this->lls_error_count += rhs.lls_error_count;
    this->lls_warning_count += rhs.lls_warning_count;
    this->lls_total_count += rhs.lls_total_count;

    return *this;
}

log_op_description&
log_op_description::operator|=(const log_op_description& rhs)
{
    if (!this->lod_id && rhs.lod_id) {
        this->lod_id = rhs.lod_id;
    }
    if (this->lod_elements.size() < rhs.lod_elements.size()) {
        this->lod_elements = rhs.lod_elements;
    }

    return *this;
}

void
opid_time_range::clear()
{
    this->otr_range.invalidate();
    this->otr_sub_ops.clear();
    this->otr_level_stats = {};
}

opid_time_range&
opid_time_range::operator|=(const opid_time_range& rhs)
{
    this->otr_range |= rhs.otr_range;
    this->otr_description |= rhs.otr_description;
    this->otr_level_stats |= rhs.otr_level_stats;
    for (const auto& rhs_sub : rhs.otr_sub_ops) {
        bool found = false;

        for (auto& sub : this->otr_sub_ops) {
            if (sub.ostr_subid == rhs_sub.ostr_subid) {
                sub.ostr_range |= rhs_sub.ostr_range;
                found = true;
            }
        }
        if (!found) {
            this->otr_sub_ops.emplace_back(rhs_sub);
        }
    }
    std::stable_sort(this->otr_sub_ops.begin(), this->otr_sub_ops.end());

    return *this;
}

void
log_level_stats::update_msg_count(log_level_t lvl, int32_t amount)
{
    switch (lvl) {
        case LEVEL_FATAL:
        case LEVEL_CRITICAL:
        case LEVEL_ERROR:
            this->lls_error_count += amount;
            break;
        case LEVEL_WARNING:
            this->lls_warning_count += amount;
            break;
        default:
            break;
    }
    this->lls_total_count += amount;
}

void
opid_time_range::close_sub_ops(const string_fragment& subid)
{
    for (auto& other_sub : this->otr_sub_ops) {
        if (other_sub.ostr_subid == subid) {
            other_sub.ostr_open = false;
        }
    }
}

log_opid_map::iterator
log_opid_state::insert_op(ArenaAlloc::Alloc<char>& alloc,
                          const string_fragment& opid,
                          const struct timeval& log_tv)
{
    auto retval = this->los_opid_ranges.find(opid);
    if (retval == this->los_opid_ranges.end()) {
        auto opid_copy = opid.to_owned(alloc);
        auto otr = opid_time_range{time_range{log_tv, log_tv}};
        auto emplace_res = this->los_opid_ranges.emplace(opid_copy, otr);
        retval = emplace_res.first;
    } else {
        retval->second.otr_range.extend_to(log_tv);
    }

    return retval;
}

opid_sub_time_range*
log_opid_state::sub_op_in_use(ArenaAlloc::Alloc<char>& alloc,
                              log_opid_map::iterator& op_iter,
                              const string_fragment& subid,
                              const timeval& tv,
                              log_level_t level)
{
    const auto& opid = op_iter->first;
    auto sub_iter = this->los_sub_in_use.find(subid);
    if (sub_iter == this->los_sub_in_use.end()) {
        auto emp_res
            = this->los_sub_in_use.emplace(subid.to_owned(alloc), opid);

        sub_iter = emp_res.first;
    }

    auto retval = sub_iter->first;
    if (sub_iter->second != opid) {
        auto other_otr
            = lnav::map::find(this->los_opid_ranges, sub_iter->second);
        if (other_otr) {
            other_otr->get().close_sub_ops(retval);
        }
    }
    sub_iter->second = opid;

    auto& otr = op_iter->second;
    auto sub_op_iter = otr.otr_sub_ops.rbegin();
    for (; sub_op_iter != otr.otr_sub_ops.rend(); ++sub_op_iter) {
        if (sub_op_iter->ostr_open && sub_op_iter->ostr_subid == retval) {
            break;
        }
    }
    if (sub_op_iter == otr.otr_sub_ops.rend()) {
        otr.otr_sub_ops.emplace_back(opid_sub_time_range{
            retval,
            time_range{tv, tv},
        });
        otr.otr_sub_ops.back().ostr_level_stats.update_msg_count(level);

        return &otr.otr_sub_ops.back();
    } else {
        sub_op_iter->ostr_range.extend_to(tv);
        sub_op_iter->ostr_level_stats.update_msg_count(level);
        return &(*sub_op_iter);
    }
}

std::optional<std::string>
log_format::opid_descriptor::matches(const string_fragment& sf) const
{
    if (this->od_extractor.pp_value) {
        thread_local auto desc_md = lnav::pcre2pp::match_data::unitialized();

        auto desc_match_res = this->od_extractor.pp_value->capture_from(sf)
                                  .into(desc_md)
                                  .matches(PCRE2_NO_UTF_CHECK | PCRE2_ANCHORED)
                                  .ignore_error();
        if (desc_match_res) {
            return desc_md.to_string();
        }

        return std::nullopt;
    }
    return sf.to_string();
}

std::string
log_format::opid_descriptors::to_string(
    const lnav::map::small<size_t, std::string>& lod) const
{
    std::string retval;

    for (size_t lpc = 0; lpc < this->od_descriptors->size(); lpc++) {
        retval.append(this->od_descriptors->at(lpc).od_prefix);
        auto iter = lod.find(lpc);
        if (iter != lod.end()) {
            retval.append(iter->second);
        }
        retval.append(this->od_descriptors->at(lpc).od_suffix);
    }

    return retval;
}

chart_type_t
logline_value_meta::to_chart_type() const
{
    auto retval = chart_type_t::hist;
    switch (this->lvm_kind) {
        case value_kind_t::VALUE_NULL:
            retval = chart_type_t::none;
            break;
        case value_kind_t::VALUE_INTEGER:
            if (!this->lvm_identifier && !this->lvm_foreign_key) {
                retval = chart_type_t::spectro;
            }
            break;
        case value_kind_t::VALUE_FLOAT:
            retval = chart_type_t::spectro;
            break;
        case value_kind_t::VALUE_XML:
        case value_kind_t::VALUE_JSON:
        case value_kind_t::VALUE_BOOLEAN:
        case value_kind_t::VALUE_TIMESTAMP:
            retval = chart_type_t::none;
            break;
        default:
            break;
    }

    return retval;
}

struct line_range
logline_value::origin_in_full_msg(const char* msg, ssize_t len) const
{
    if (this->lv_sub_offset == 0) {
        return this->lv_origin;
    }

    if (len == -1) {
        len = strlen(msg);
    }

    struct line_range retval = this->lv_origin;
    const char *last = msg, *msg_end = msg + len;

    for (int lpc = 0; lpc < this->lv_sub_offset; lpc++) {
        const auto* next = (const char*) memchr(last, '\n', msg_end - last);
        require(next != nullptr);

        next += 1;
        int amount = (next - last);

        retval.lr_start += amount;
        if (retval.lr_end != -1) {
            retval.lr_end += amount;
        }

        last = next + 1;
    }

    if (retval.lr_end == -1) {
        const auto* eol = (const char*) memchr(last, '\n', msg_end - last);

        if (eol == nullptr) {
            retval.lr_end = len;
        } else {
            retval.lr_end = eol - msg;
        }
    }

    return retval;
}

logline_value::logline_value(logline_value_meta lvm,
                             shared_buffer_ref& sbr,
                             struct line_range origin)
    : lv_meta(std::move(lvm)), lv_origin(origin)
{
    if (sbr.get_data() == nullptr) {
        this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL;
    }

    switch (this->lv_meta.lvm_kind) {
        case value_kind_t::VALUE_JSON:
        case value_kind_t::VALUE_XML:
        case value_kind_t::VALUE_STRUCT:
        case value_kind_t::VALUE_TEXT:
        case value_kind_t::VALUE_QUOTED:
        case value_kind_t::VALUE_W3C_QUOTED:
        case value_kind_t::VALUE_TIMESTAMP:
            require(origin.lr_end != -1);
            this->lv_frag = string_fragment::from_byte_range(
                sbr.get_data(), origin.lr_start, origin.lr_end);
            break;

        case value_kind_t::VALUE_NULL:
            break;

        case value_kind_t::VALUE_INTEGER: {
            auto scan_res
                = scn::scan_value<int64_t>(sbr.to_string_view(origin));
            if (scan_res) {
                this->lv_value.i = scan_res->value();
            } else {
                this->lv_value.i = 0;
            }
            break;
        }

        case value_kind_t::VALUE_FLOAT: {
            auto scan_res = scn::scan_value<double>(sbr.to_string_view(origin));
            if (scan_res) {
                this->lv_value.d = scan_res->value();
            } else {
                this->lv_value.d = 0;
            }
            break;
        }

        case value_kind_t::VALUE_BOOLEAN:
            if (strncmp(
                    sbr.get_data_at(origin.lr_start), "true", origin.length())
                    == 0
                || strncmp(
                       sbr.get_data_at(origin.lr_start), "yes", origin.length())
                    == 0)
            {
                this->lv_value.i = 1;
            } else {
                this->lv_value.i = 0;
            }
            break;

        case value_kind_t::VALUE_UNKNOWN:
        case value_kind_t::VALUE__MAX:
            ensure(0);
            break;
    }
}

std::string
logline_value::to_string() const
{
    char buffer[128];

    switch (this->lv_meta.lvm_kind) {
        case value_kind_t::VALUE_NULL:
            return "null";

        case value_kind_t::VALUE_JSON:
        case value_kind_t::VALUE_XML:
        case value_kind_t::VALUE_STRUCT:
        case value_kind_t::VALUE_TEXT:
        case value_kind_t::VALUE_TIMESTAMP:
            if (this->lv_str) {
                return this->lv_str.value();
            }
            if (this->lv_frag.empty()) {
                return this->lv_intern_string.to_string();
            }
            return this->lv_frag.to_string();

        case value_kind_t::VALUE_QUOTED:
        case value_kind_t::VALUE_W3C_QUOTED:
            if (this->lv_frag.empty()) {
                return "";
            } else {
                switch (this->lv_frag.data()[0]) {
                    case '\'':
                    case '"': {
                        auto unquote_func = this->lv_meta.lvm_kind
                                == value_kind_t::VALUE_W3C_QUOTED
                            ? unquote_w3c
                            : unquote;
                        stack_buf allocator;
                        auto* unquoted_str
                            = allocator.allocate(this->lv_frag.length());
                        size_t unquoted_len;

                        unquoted_len = unquote_func(unquoted_str,
                                                    this->lv_frag.data(),
                                                    this->lv_frag.length());
                        return {unquoted_str, unquoted_len};
                    }
                    default:
                        return this->lv_frag.to_string();
                }
            }
            break;

        case value_kind_t::VALUE_INTEGER:
            snprintf(buffer, sizeof(buffer), "%" PRId64, this->lv_value.i);
            break;

        case value_kind_t::VALUE_FLOAT:
            snprintf(buffer, sizeof(buffer), "%lf", this->lv_value.d);
            break;

        case value_kind_t::VALUE_BOOLEAN:
            if (this->lv_value.i) {
                return "true";
            } else {
                return "false";
            }
            break;
        case value_kind_t::VALUE_UNKNOWN:
        case value_kind_t::VALUE__MAX:
            ensure(0);
            break;
    }

    return {buffer};
}

string_fragment
logline_value::to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const
{
    char buffer[128];

    switch (this->lv_meta.lvm_kind) {
        case value_kind_t::VALUE_NULL:
            return "null"_frag;

        case value_kind_t::VALUE_JSON:
        case value_kind_t::VALUE_XML:
        case value_kind_t::VALUE_STRUCT:
        case value_kind_t::VALUE_TEXT:
        case value_kind_t::VALUE_TIMESTAMP:
            if (this->lv_str) {
                return string_fragment::from_str(this->lv_str.value())
                    .to_owned(alloc);
            }
            if (this->lv_frag.empty()) {
                return this->lv_intern_string.to_string_fragment().to_owned(
                    alloc);
            }
            return this->lv_frag.to_owned(alloc);

        case value_kind_t::VALUE_QUOTED:
        case value_kind_t::VALUE_W3C_QUOTED:
            if (this->lv_frag.empty()) {
                return string_fragment{};
            } else {
                switch (this->lv_frag.data()[0]) {
                    case '\'':
                    case '"': {
                        auto unquote_func = this->lv_meta.lvm_kind
                                == value_kind_t::VALUE_W3C_QUOTED
                            ? unquote_w3c
                            : unquote;
                        stack_buf allocator;
                        auto* unquoted_str
                            = allocator.allocate(this->lv_frag.length());
                        size_t unquoted_len;

                        unquoted_len = unquote_func(unquoted_str,
                                                    this->lv_frag.data(),
                                                    this->lv_frag.length());
                        return string_fragment::from_bytes(unquoted_str,
                                                           unquoted_len)
                            .to_owned(alloc);
                    }
                    default:
                        return this->lv_frag.to_owned(alloc);
                }
            }
            break;

        case value_kind_t::VALUE_INTEGER:
            snprintf(buffer, sizeof(buffer), "%" PRId64, this->lv_value.i);
            break;

        case value_kind_t::VALUE_FLOAT:
            snprintf(buffer, sizeof(buffer), "%lf", this->lv_value.d);
            break;

        case value_kind_t::VALUE_BOOLEAN:
            if (this->lv_value.i) {
                return "true"_frag;
            }
            return "false"_frag;
            break;
        case value_kind_t::VALUE_UNKNOWN:
        case value_kind_t::VALUE__MAX:
            ensure(0);
            break;
    }

    return string_fragment::from_c_str(buffer).to_owned(alloc);
}

std::vector<std::shared_ptr<log_format>> log_format::lf_root_formats;

std::vector<std::shared_ptr<log_format>>&
log_format::get_root_formats()
{
    return lf_root_formats;
}

void
external_log_format::update_op_description(
    const std::map<intern_string_t, opid_descriptors>& desc_defs,
    log_op_description& lod,
    const pattern* fpat,
    const lnav::pcre2pp::match_data& md)
{
    std::optional<std::string> desc_elem_str;
    if (!lod.lod_id) {
        for (const auto& desc_def_pair : desc_defs) {
            if (lod.lod_id) {
                break;
            }
            for (const auto& desc_def : *desc_def_pair.second.od_descriptors) {
                auto desc_field_index_iter = fpat->p_value_name_to_index.find(
                    desc_def.od_field.pp_value);

                if (desc_field_index_iter == fpat->p_value_name_to_index.end())
                {
                    continue;
                }
                auto desc_cap_opt = md[desc_field_index_iter->second];

                if (!desc_cap_opt) {
                    continue;
                }

                desc_elem_str = desc_def.matches(desc_cap_opt.value());
                if (desc_elem_str) {
                    lod.lod_id = desc_def_pair.first;
                }
            }
        }
    }
    if (lod.lod_id) {
        const auto& desc_def_v
            = *desc_defs.find(lod.lod_id.value())->second.od_descriptors;
        auto& desc_v = lod.lod_elements;

        if (desc_def_v.size() == desc_v.size()) {
            return;
        }
        for (size_t desc_def_index = 0; desc_def_index < desc_def_v.size();
             desc_def_index++)
        {
            const auto& desc_def = desc_def_v[desc_def_index];
            auto found_desc = desc_v.begin();

            for (; found_desc != desc_v.end(); ++found_desc) {
                if (found_desc->first == desc_def_index) {
                    break;
                }
            }
            auto desc_field_index_iter
                = fpat->p_value_name_to_index.find(desc_def.od_field.pp_value);

            if (desc_field_index_iter == fpat->p_value_name_to_index.end()) {
                continue;
            }
            auto desc_cap_opt = md[desc_field_index_iter->second];
            if (!desc_cap_opt) {
                continue;
            }

            if (!desc_elem_str) {
                desc_elem_str = desc_def.matches(desc_cap_opt.value());
            }
            if (desc_elem_str) {
                if (found_desc == desc_v.end()) {
                    desc_v.emplace_back(desc_def_index, desc_elem_str.value());
                } else if (!desc_elem_str->empty()) {
                    found_desc->second.append(desc_def.od_joiner);
                    found_desc->second.append(desc_elem_str.value());
                }
            }
            desc_elem_str = std::nullopt;
        }
    }
}

void
external_log_format::update_op_description(
    const std::map<intern_string_t, opid_descriptors>& desc_defs,
    log_op_description& lod)
{
    std::optional<std::string> desc_elem_str;
    if (!lod.lod_id) {
        for (const auto& desc_def_pair : desc_defs) {
            if (lod.lod_id) {
                break;
            }
            for (const auto& desc_def : *desc_def_pair.second.od_descriptors) {
                auto desc_cap_iter
                    = this->lf_desc_captures.find(desc_def.od_field.pp_value);

                if (desc_cap_iter == this->lf_desc_captures.end()) {
                    continue;
                }
                desc_elem_str = desc_def.matches(desc_cap_iter->second);
                if (desc_elem_str) {
                    lod.lod_id = desc_def_pair.first;
                }
            }
        }
    }
    if (lod.lod_id) {
        const auto& desc_def_v
            = *desc_defs.find(lod.lod_id.value())->second.od_descriptors;
        auto& desc_v = lod.lod_elements;

        if (desc_def_v.size() != desc_v.size()) {
            for (size_t desc_def_index = 0; desc_def_index < desc_def_v.size();
                 desc_def_index++)
            {
                const auto& desc_def = desc_def_v[desc_def_index];
                auto found_desc = desc_v.begin();

                for (; found_desc != desc_v.end(); ++found_desc) {
                    if (found_desc->first == desc_def_index) {
                        break;
                    }
                }
                auto desc_cap_iter
                    = this->lf_desc_captures.find(desc_def.od_field.pp_value);
                if (desc_cap_iter == this->lf_desc_captures.end()) {
                    continue;
                }

                if (!desc_elem_str) {
                    desc_elem_str = desc_def.matches(desc_cap_iter->second);
                }
                if (desc_elem_str) {
                    if (found_desc == desc_v.end()) {
                        desc_v.emplace_back(desc_def_index,
                                            desc_elem_str.value());
                    } else if (!desc_elem_str->empty()) {
                        found_desc->second.append(desc_def.od_joiner);
                        found_desc->second.append(desc_elem_str.value());
                    }
                }
                desc_elem_str = std::nullopt;
            }
        }
    }
}

static bool
next_format(
    const std::vector<std::shared_ptr<external_log_format::pattern>>& patterns,
    int& index,
    int& locked_index)
{
    bool retval = true;

    if (locked_index == -1) {
        index += 1;
        if (index >= (int) patterns.size()) {
            retval = false;
        }
    } else if (index == locked_index) {
        retval = false;
    } else {
        index = locked_index;
    }

    return retval;
}

bool
log_format::next_format(const pcre_format* fmt, int& index, int& locked_index)
{
    bool retval = true;

    if (locked_index == -1) {
        index += 1;
        if (fmt[index].name == nullptr) {
            retval = false;
        }
    } else if (index == locked_index) {
        retval = false;
    } else {
        index = locked_index;
    }

    return retval;
}

const char*
log_format::log_scanf(uint32_t line_number,
                      string_fragment line,
                      const pcre_format* fmt,
                      const char* time_fmt[],
                      struct exttm* tm_out,
                      struct timeval* tv_out,

                      string_fragment* ts_out,
                      std::optional<string_fragment>* level_out)
{
    int curr_fmt = -1;
    const char* retval = nullptr;
    bool done = false;
    int pat_index = this->last_pattern_index();

    while (!done && next_format(fmt, curr_fmt, pat_index)) {
        thread_local auto md = lnav::pcre2pp::match_data::unitialized();

        auto match_res = fmt[curr_fmt]
                             .pcre->capture_from(line)
                             .into(md)
                             .matches(PCRE2_NO_UTF_CHECK)
                             .ignore_error();
        if (!match_res) {
            retval = nullptr;
        } else {
            auto ts = md[fmt[curr_fmt].pf_timestamp_index];

            retval = this->lf_date_time.scan(
                ts->data(), ts->length(), nullptr, tm_out, *tv_out);

            if (retval == nullptr) {
                auto ls = this->lf_date_time.unlock();
                retval = this->lf_date_time.scan(
                    ts->data(), ts->length(), nullptr, tm_out, *tv_out);
                if (retval != nullptr) {
                    auto old_flags
                        = this->lf_timestamp_flags & DATE_TIME_SET_FLAGS;
                    auto new_flags = tm_out->et_flags & DATE_TIME_SET_FLAGS;

                    // It is unlikely a valid timestamp would lose much
                    // precision.
                    if (new_flags != old_flags) {
                        retval = nullptr;
                    }
                }
                if (retval == nullptr) {
                    this->lf_date_time.relock(ls);
                } else {
                    log_debug(
                        "%d: changed time format to '%s' due to %.*s",
                        line_number,
                        PTIMEC_FORMAT_STR[this->lf_date_time.dts_fmt_lock],
                        ts->length(),
                        ts->data());
                }
            }

            if (retval) {
                *ts_out = ts.value();
                *level_out = md[2];
                if (curr_fmt != pat_index) {
                    uint32_t lock_line;

                    if (this->lf_pattern_locks.empty()) {
                        lock_line = 0;
                    } else {
                        lock_line = line_number;
                    }

                    this->lf_pattern_locks.emplace_back(lock_line, curr_fmt);
                }
                this->lf_timestamp_flags = tm_out->et_flags;
                done = true;
            }
        }
    }

    return retval;
}

void
log_format::annotate(logfile* lf,
                     uint64_t line_number,
                     string_attrs_t& sa,
                     logline_value_vector& values,
                     bool annotate_module) const
{
    if (lf != nullptr && !values.lvv_opid_value) {
        const auto& bm = lf->get_bookmark_metadata();
        auto bm_iter = bm.find(line_number);
        if (bm_iter != bm.end() && !bm_iter->second.bm_opid.empty()) {
            values.lvv_opid_value = bm_iter->second.bm_opid;
            values.lvv_opid_provenance
                = logline_value_vector::opid_provenance::user;
        }
    }
}

void
log_format::check_for_new_year(std::vector<logline>& dst,
                               exttm etm,
                               struct timeval log_tv)
{
    if (dst.empty()) {
        return;
    }

    time_t diff
        = dst.back().get_time<std::chrono::seconds>().count() - log_tv.tv_sec;
    int off_year = 0, off_month = 0, off_day = 0, off_hour = 0;
    bool do_change = true;

    if (diff <= 0) {
        return;
    }
    if ((etm.et_flags & ETF_MONTH_SET) && diff >= (24 * 60 * 60)) {
        off_year = 1;
    } else if (diff >= (24 * 60 * 60)) {
        off_month = 1;
    } else if (!(etm.et_flags & ETF_DAY_SET) && (diff >= (60 * 60))) {
        off_day = 1;
    } else if (!(etm.et_flags & ETF_HOUR_SET) && (diff >= 60)) {
        off_hour = 1;
    } else {
        do_change = false;
    }

    if (!do_change) {
        return;
    }
    log_debug("%d:detected time rollover; offsets=%d %d %d %d",
              dst.size(),
              off_year,
              off_month,
              off_day,
              off_hour);
    for (auto& ll : dst) {
        time_t ot = ll.get_time<std::chrono::seconds>().count();
        struct tm otm;

        gmtime_r(&ot, &otm);
        otm.tm_yday = -1;
        if (otm.tm_year < off_year) {
            otm.tm_year = 0;
        } else {
            otm.tm_year -= off_year;
        }
        otm.tm_mon -= off_month;
        if (otm.tm_mon < 0) {
            otm.tm_mon += 12;
        }
        auto new_time = tm2sec(&otm);
        if (new_time == -1) {
            continue;
        }
        new_time -= (off_day * 24 * 60 * 60) + (off_hour * 60 * 60);
        auto old_sub = ll.get_subsecond_time<std::chrono::microseconds>();
        ll.set_time(std::chrono::seconds{new_time});
        ll.set_subsecond_time(old_sub);
    }
}

/*
 * XXX This needs some cleanup.
 */
struct json_log_userdata {
    json_log_userdata(shared_buffer_ref& sbr, scan_batch_context* sbc)
        : jlu_shared_buffer(sbr), jlu_batch_context(sbc)
    {
    }

    const external_log_format::value_def* get_field_def(
        yajlpp_parse_context* ypc)
    {
        const auto field_frag = ypc->get_path_as_string_fragment();
        auto* format = this->jlu_format;

        if (this->jlu_read_order_index < format->elf_value_def_read_order.size()
            && format->elf_value_def_read_order[this->jlu_read_order_index]
                    .first
                == field_frag)
        {
            return format
                ->elf_value_def_read_order[this->jlu_read_order_index++]
                .second;
        }

        format->elf_value_def_read_order.resize(this->jlu_read_order_index);
        auto vd_iter = format->elf_value_def_frag_map.find(field_frag);
        if (vd_iter != format->elf_value_def_frag_map.end()) {
            format->elf_value_def_read_order.emplace_back(vd_iter->first,
                                                          vd_iter->second);
            this->jlu_read_order_index += 1;
            return vd_iter->second;
        }

        auto owned_frag = field_frag.to_owned(format->elf_allocator);
        format->elf_value_def_frag_map[owned_frag] = nullptr;
        format->elf_value_def_read_order.emplace_back(owned_frag, nullptr);
        this->jlu_read_order_index += 1;
        return nullptr;
    }

    void add_sub_lines_for(const external_log_format::value_def* vd,
                           bool top_level,
                           std::optional<double> val,
                           const unsigned char* str,
                           ssize_t len,
                           yajl_string_props_t* props)
    {
        auto res = this->jlu_format->value_line_count(
            vd, top_level, val, str, len, props);
        this->jlu_has_ansi |= res.vlcr_has_ansi;
        if (!res.vlcr_valid_utf) {
            this->jlu_valid_utf = false;
        }
        this->jlu_sub_line_count += res.vlcr_count;
        this->jlu_quality += res.vlcr_line_format_count;
        if (res.vlcr_line_format_index) {
            this->jlu_format_hits[res.vlcr_line_format_index.value()] = true;
        }
    }

    external_log_format* jlu_format{nullptr};
    const logline* jlu_line{nullptr};
    logline* jlu_base_line{nullptr};
    int jlu_sub_line_count{1};
    bool jlu_has_ansi{false};
    bool jlu_valid_utf{true};
    yajl_handle jlu_handle{nullptr};
    const char* jlu_line_value{nullptr};
    size_t jlu_line_size{0};
    size_t jlu_sub_start{0};
    uint32_t jlu_quality{0};
    uint32_t jlu_strikes{0};
    std::vector<bool> jlu_format_hits;
    shared_buffer_ref& jlu_shared_buffer;
    scan_batch_context* jlu_batch_context;
    std::optional<string_fragment> jlu_opid_frag;
    std::optional<std::string> jlu_subid;
    exttm jlu_exttm;
    size_t jlu_read_order_index{0};
    subline_options jlu_subline_opts;
};

static int read_json_field(yajlpp_parse_context* ypc,
                           const unsigned char* str,
                           size_t len,
                           yajl_string_props_t*);

static int
read_json_null(yajlpp_parse_context* ypc)
{
    json_log_userdata* jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    jlu->add_sub_lines_for(
        vd, ypc->is_level(1), std::nullopt, nullptr, -1, nullptr);

    return 1;
}

static int
read_json_bool(yajlpp_parse_context* ypc, int val)
{
    json_log_userdata* jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    jlu->add_sub_lines_for(
        vd, ypc->is_level(1), std::nullopt, nullptr, -1, nullptr);

    return 1;
}

static int
read_json_number(yajlpp_parse_context* ypc,
                 const char* numberVal,
                 size_t numberLen)
{
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    auto number_frag = string_fragment::from_bytes(numberVal, numberLen);
    std::optional<double> val;

    intern_string_t field_name;
    const auto* vd = jlu->get_field_def(ypc);
    if (vd != nullptr) {
        field_name = vd->vd_meta.lvm_name;
    }

    if (field_name.empty()) {
    } else if (jlu->jlu_format->lf_timestamp_field == field_name) {
        long long divisor = jlu->jlu_format->elf_timestamp_divisor;
        auto scan_res = scn::scan_value<double>(number_frag.to_string_view());
        if (!scan_res) {
            log_error("invalid number %.*s", numberLen, numberVal);
            return 0;
        }
        auto ts_val = scan_res.value().value();
        timeval tv;
        tv.tv_sec = ts_val / divisor;
        tv.tv_usec = fmod(ts_val, divisor) * (1000000.0 / divisor);
        jlu->jlu_format->lf_date_time.to_localtime(tv.tv_sec, jlu->jlu_exttm);
        tv.tv_sec = tm2sec(&jlu->jlu_exttm.et_tm);
        jlu->jlu_exttm.et_gmtoff
            = jlu->jlu_format->lf_date_time.dts_local_offset_cache;
        jlu->jlu_exttm.et_flags
            |= ETF_MACHINE_ORIENTED | ETF_SUB_NOT_IN_FORMAT | ETF_ZONE_SET;
        if (divisor == 1000) {
            jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
        } else {
            jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
        }
        jlu->jlu_exttm.et_nsec = tv.tv_usec * 1000;
        jlu->jlu_base_line->set_time(tv);
    } else if (jlu->jlu_format->lf_subsecond_field == field_name) {
        auto scan_res = scn::scan_value<double>(number_frag.to_string_view());
        if (!scan_res) {
            log_error("invalid number %.*s", numberLen, numberVal);
            return 0;
        }
        auto ts_val = scan_res.value().value();

        uint64_t millis = 0;
        jlu->jlu_exttm.et_flags &= ~(ETF_MICROS_SET | ETF_MILLIS_SET);
        switch (jlu->jlu_format->lf_subsecond_unit.value()) {
            case log_format::subsecond_unit::milli:
                millis = ts_val;
                jlu->jlu_exttm.et_nsec = ts_val * 1000000;
                jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
                break;
            case log_format::subsecond_unit::micro:
                millis = std::chrono::duration_cast<std::chrono::milliseconds>(
                             std::chrono::microseconds((int64_t) ts_val))
                             .count();
                jlu->jlu_exttm.et_nsec = ts_val * 1000;
                jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
                break;
            case log_format::subsecond_unit::nano:
                millis = std::chrono::duration_cast<std::chrono::milliseconds>(
                             std::chrono::nanoseconds((int64_t) ts_val))
                             .count();
                jlu->jlu_exttm.et_nsec = ts_val;
                jlu->jlu_exttm.et_flags |= ETF_NANOS_SET;
                break;
        }
        jlu->jlu_exttm.et_flags |= ETF_SUB_NOT_IN_FORMAT;
        jlu->jlu_base_line->set_subsecond_time(
            std::chrono::milliseconds(millis));
    } else if (jlu->jlu_format->elf_level_field == field_name) {
        if (jlu->jlu_format->elf_level_pairs.empty()) {
            jlu->jlu_base_line->set_level(jlu->jlu_format->convert_level(
                number_frag, jlu->jlu_batch_context));
        } else {
            auto scan_res
                = scn::scan_int<int64_t>(number_frag.to_string_view());
            if (!scan_res) {
                log_error("invalid number %.*s", numberLen, numberVal);
                return 0;
            }
            auto level_int = scan_res.value().value();

            for (const auto& pair : jlu->jlu_format->elf_level_pairs) {
                if (pair.first == level_int) {
                    jlu->jlu_base_line->set_level(pair.second);
                    break;
                }
            }
        }
    } else if (vd != nullptr) {
        if ((vd->vd_meta.lvm_kind == value_kind_t::VALUE_INTEGER
             || vd->vd_meta.lvm_kind == value_kind_t::VALUE_FLOAT)
            && !vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier)
        {
            auto scan_res
                = scn::scan_value<double>(number_frag.to_string_view());
            if (!scan_res) {
                log_error("invalid number %.*s", numberLen, numberVal);
                return 0;
            }
            val = scan_res.value().value();
        }
    }

    jlu->add_sub_lines_for(vd,
                           ypc->is_level(1),
                           val,
                           (const unsigned char*) numberVal,
                           numberLen,
                           nullptr);

    return 1;
}

static int
json_array_start(void* ctx)
{
    auto* ypc = (yajlpp_parse_context*) ctx;
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;

    if (ypc->ypc_path_index_stack.size() == 2) {
        const intern_string_t field_name = ypc->get_path_fragment_i(0);

        external_log_format::value_def* vd = nullptr;
        auto iter = jlu->jlu_format->elf_value_defs.find(field_name);
        if (iter != jlu->jlu_format->elf_value_defs.end()) {
            vd = iter->second.get();
        }
        jlu->add_sub_lines_for(vd, true, std::nullopt, nullptr, -1, nullptr);
        jlu->jlu_sub_start = yajl_get_bytes_consumed(jlu->jlu_handle) - 1;
    }

    return 1;
}

static int
json_array_end(void* ctx)
{
    yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx;
    json_log_userdata* jlu = (json_log_userdata*) ypc->ypc_userdata;

    if (ypc->ypc_path_index_stack.size() == 1) {
        const intern_string_t field_name = ypc->get_path_fragment_i(0);
        size_t sub_end = yajl_get_bytes_consumed(jlu->jlu_handle);
        jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
            jlu->jlu_format->get_value_meta(field_name,
                                            value_kind_t::VALUE_JSON),
            string_fragment::from_byte_range(jlu->jlu_shared_buffer.get_data(),
                                             jlu->jlu_sub_start,
                                             sub_end));
    }

    return 1;
}

static const json_path_container json_log_handlers = {
    yajlpp::pattern_property_handler("\\w+")
        .add_cb(read_json_null)
        .add_cb(read_json_bool)
        .add_cb(read_json_number)
        .add_cb(read_json_field),
};

static int rewrite_json_field(yajlpp_parse_context* ypc,
                              const unsigned char* str,
                              size_t len,
                              yajl_string_props_t*);

static int
rewrite_json_null(yajlpp_parse_context* ypc)
{
    auto jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    if (!ypc->is_level(1) && vd == nullptr) {
        return 1;
    }
    jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
        jlu->jlu_format->get_value_meta(ypc, vd, value_kind_t::VALUE_NULL));

    return 1;
}

static int
rewrite_json_bool(yajlpp_parse_context* ypc, int val)
{
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    if (!ypc->is_level(1) && vd == nullptr) {
        return 1;
    }
    jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
        jlu->jlu_format->get_value_meta(ypc, vd, value_kind_t::VALUE_BOOLEAN),
        (bool) val);
    return 1;
}

static int
rewrite_json_int(yajlpp_parse_context* ypc, long long val)
{
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    if (vd != nullptr) {
        const intern_string_t field_name = vd->vd_meta.lvm_name;
        if (jlu->jlu_format->lf_timestamp_field == field_name) {
            long long divisor = jlu->jlu_format->elf_timestamp_divisor;
            timeval tv;

            tv.tv_sec = val / divisor;
            tv.tv_usec = fmod(val, divisor) * (1000000.0 / divisor);
            jlu->jlu_format->lf_date_time.to_localtime(tv.tv_sec,
                                                       jlu->jlu_exttm);
            jlu->jlu_exttm.et_gmtoff
                = jlu->jlu_format->lf_date_time.dts_local_offset_cache;
            jlu->jlu_exttm.et_flags |= ETF_MACHINE_ORIENTED
                | ETF_SUB_NOT_IN_FORMAT | ETF_ZONE_SET | ETF_Z_FOR_UTC;
            if (divisor == 1) {
                jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
            } else {
                jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
            }
            jlu->jlu_exttm.et_nsec = tv.tv_usec * 1000;
        } else if (jlu->jlu_format->lf_subsecond_field == field_name) {
            jlu->jlu_exttm.et_flags &= ~(ETF_MICROS_SET | ETF_MILLIS_SET);
            switch (jlu->jlu_format->lf_subsecond_unit.value()) {
                case log_format::subsecond_unit::milli:
                    jlu->jlu_exttm.et_nsec = val * 1000000;
                    jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
                    break;
                case log_format::subsecond_unit::micro:
                    jlu->jlu_exttm.et_nsec = val * 1000;
                    jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
                    break;
                case log_format::subsecond_unit::nano:
                    jlu->jlu_exttm.et_nsec = val;
                    jlu->jlu_exttm.et_flags |= ETF_NANOS_SET;
                    break;
            }
            jlu->jlu_exttm.et_flags |= ETF_SUB_NOT_IN_FORMAT;
        }
    }

    if (!ypc->is_level(1) && vd == nullptr) {
        return 1;
    }
    jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
        jlu->jlu_format->get_value_meta(ypc, vd, value_kind_t::VALUE_INTEGER),
        (int64_t) val);
    return 1;
}

static int
rewrite_json_double(yajlpp_parse_context* ypc, double val)
{
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    const auto* vd = jlu->get_field_def(ypc);

    if (vd != nullptr) {
        const intern_string_t field_name = vd->vd_meta.lvm_name;
        if (jlu->jlu_format->lf_timestamp_field == field_name) {
            long long divisor = jlu->jlu_format->elf_timestamp_divisor;
            timeval tv;

            tv.tv_sec = val / divisor;
            tv.tv_usec = fmod(val, divisor) * (1000000.0 / divisor);
            jlu->jlu_format->lf_date_time.to_localtime(tv.tv_sec,
                                                       jlu->jlu_exttm);
            jlu->jlu_exttm.et_gmtoff
                = jlu->jlu_format->lf_date_time.dts_local_offset_cache;
            jlu->jlu_exttm.et_flags |= ETF_MACHINE_ORIENTED
                | ETF_SUB_NOT_IN_FORMAT | ETF_ZONE_SET | ETF_Z_FOR_UTC;
            if (divisor == 1) {
                jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
            } else {
                jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
            }
            jlu->jlu_exttm.et_nsec = tv.tv_usec * 1000;
        } else if (jlu->jlu_format->lf_subsecond_field == field_name) {
            jlu->jlu_exttm.et_flags &= ~(ETF_MICROS_SET | ETF_MILLIS_SET);
            switch (jlu->jlu_format->lf_subsecond_unit.value()) {
                case log_format::subsecond_unit::milli:
                    jlu->jlu_exttm.et_nsec = val * 1000000;
                    jlu->jlu_exttm.et_flags |= ETF_MILLIS_SET;
                    break;
                case log_format::subsecond_unit::micro:
                    jlu->jlu_exttm.et_nsec = val * 1000;
                    jlu->jlu_exttm.et_flags |= ETF_MICROS_SET;
                    break;
                case log_format::subsecond_unit::nano:
                    jlu->jlu_exttm.et_nsec = val;
                    jlu->jlu_exttm.et_flags |= ETF_NANOS_SET;
                    break;
            }
            jlu->jlu_exttm.et_flags |= ETF_SUB_NOT_IN_FORMAT;
        }
    }

    if (!ypc->is_level(1) && vd == nullptr) {
        return 1;
    }
    jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
        jlu->jlu_format->get_value_meta(ypc, vd, value_kind_t::VALUE_FLOAT),
        val);

    return 1;
}

static const json_path_container json_log_rewrite_handlers = {
    yajlpp::pattern_property_handler("\\w+")
        .add_cb(rewrite_json_null)
        .add_cb(rewrite_json_bool)
        .add_cb(rewrite_json_int)
        .add_cb(rewrite_json_double)
        .add_cb(rewrite_json_field),
};

bool
external_log_format::scan_for_partial(shared_buffer_ref& sbr,
                                      size_t& len_out) const
{
    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
        return false;
    }

    const auto& pat = this->elf_pattern_order[this->last_pattern_index()];
    if (!this->lf_multiline) {
        len_out = pat->p_pcre.pp_value->match_partial(sbr.to_string_fragment());
        return true;
    }

    if (pat->p_timestamp_end == -1 || pat->p_timestamp_end > (int) sbr.length())
    {
        len_out = 0;
        return false;
    }

    len_out = pat->p_pcre.pp_value->match_partial(sbr.to_string_fragment());
    return (int) len_out > pat->p_timestamp_end;
}

std::vector<lnav::console::snippet>
external_log_format::get_snippets() const
{
    std::vector<lnav::console::snippet> retval;

    for (const auto& src_pair : this->elf_format_sources) {
        retval.emplace_back(lnav::console::snippet::from(src_pair.first, "")
                                .with_line(src_pair.second));
    }

    return retval;
}

log_format::scan_result_t
external_log_format::scan_json(std::vector<logline>& dst,
                               const line_info& li,
                               shared_buffer_ref& sbr,
                               scan_batch_context& sbc)
{
    logline ll(
        li.li_file_range.fr_offset, std::chrono::microseconds{0}, LEVEL_INFO);
    auto line_frag = sbr.to_string_fragment();

    if (!line_frag.startswith("{")) {
        if (!this->lf_specialized) {
            return scan_no_match{"line is not a JSON object"};
        }

        ll.set_time(dst.back().get_time<std::chrono::microseconds>());
        ll.set_level(LEVEL_INVALID);
        dst.emplace_back(ll);
        return scan_match{0};
    }

    auto& ypc = *(this->jlf_parse_context);
    yajl_handle handle = this->jlf_yajl_handle.get();
    json_log_userdata jlu(sbr, &sbc);

    if (li.li_partial) {
        log_debug("skipping partial line at offset %d",
                  li.li_file_range.fr_offset);
        if (this->lf_specialized) {
            if (!dst.empty()) {
                ll.set_time(dst.back().get_time<std::chrono::microseconds>());
            }
            ll.set_level(LEVEL_INVALID);
            dst.emplace_back(ll);
        }
        return scan_incomplete{};
    }

    const auto* line_data = (const unsigned char*) sbr.get_data();

    this->lf_desc_captures.clear();
    this->lf_desc_allocator.reset();

    yajl_reset(handle);
    ypc.set_static_handler(json_log_handlers.jpc_children[0]);
    ypc.ypc_userdata = &jlu;
    ypc.ypc_ignore_unused = true;
    ypc.ypc_alt_callbacks.yajl_start_array = json_array_start;
    ypc.ypc_alt_callbacks.yajl_start_map = json_array_start;
    ypc.ypc_alt_callbacks.yajl_end_array = nullptr;
    ypc.ypc_alt_callbacks.yajl_end_map = nullptr;
    jlu.jlu_format = this;
    jlu.jlu_base_line = &ll;
    jlu.jlu_line_value = sbr.get_data();
    jlu.jlu_line_size = sbr.length();
    jlu.jlu_handle = handle;
    jlu.jlu_format_hits.resize(this->jlf_line_format.size());
    if (yajl_parse(handle, line_data, sbr.length()) == yajl_status_ok
        && yajl_complete_parse(handle) == yajl_status_ok)
    {
        if (ll.get_time<std::chrono::microseconds>().count() == 0) {
            if (this->lf_specialized) {
                if (!dst.empty()) {
                    ll.set_time(
                        dst.back().get_time<std::chrono::microseconds>());
                }
                ll.set_ignore(true);
                dst.emplace_back(ll);
                return scan_match{0};
            }

            return scan_no_match{
                "JSON message does not have expected timestamp property"};
        }

        if (jlu.jlu_opid_frag) {
            this->jlf_line_values.lvv_opid_value
                = jlu.jlu_opid_frag->to_string();
            this->jlf_line_values.lvv_opid_provenance
                = logline_value_vector::opid_provenance::file;
            auto opid_iter = sbc.sbc_opids.insert_op(
                sbc.sbc_allocator, jlu.jlu_opid_frag.value(), ll.get_timeval());
            opid_iter->second.otr_level_stats.update_msg_count(
                ll.get_msg_level());

            if (jlu.jlu_subid) {
                auto subid_frag
                    = string_fragment::from_str(jlu.jlu_subid.value());

                auto* ostr = sbc.sbc_opids.sub_op_in_use(sbc.sbc_allocator,
                                                         opid_iter,
                                                         subid_frag,
                                                         ll.get_timeval(),
                                                         ll.get_msg_level());
                if (ostr != nullptr && ostr->ostr_description.empty()) {
                    log_op_description sub_desc;
                    this->update_op_description(*this->lf_subid_description_def,
                                                sub_desc);
                    if (!sub_desc.lod_elements.empty()) {
                        auto& sub_desc_def = this->lf_subid_description_def->at(
                            sub_desc.lod_id.value());
                        ostr->ostr_description
                            = sub_desc_def.to_string(sub_desc.lod_elements);
                    }
                }
            }

            auto& otr = opid_iter->second;
            this->update_op_description(*this->lf_opid_description_def,
                                        otr.otr_description);
        } else {
            this->jlf_line_values.lvv_opid_value = std::nullopt;
        }

        jlu.jlu_sub_line_count += this->jlf_line_format_init_count;
        for (int lpc = 0; lpc < jlu.jlu_sub_line_count; lpc++) {
            ll.set_sub_offset(lpc);
            if (lpc > 0) {
                ll.set_level(
                    (log_level_t) (ll.get_level_and_flags() | LEVEL_CONTINUED));
            }
            ll.set_has_ansi(jlu.jlu_has_ansi);
            ll.set_valid_utf(jlu.jlu_valid_utf);
            dst.emplace_back(ll);
        }

        if (!this->lf_specialized) {
            static const intern_string_t ts_field
                = intern_string::lookup("__timestamp__", -1);
            static const intern_string_t level_field
                = intern_string::lookup("__level__");
            for (const auto& [index, jfe] :
                 lnav::itertools::enumerate(this->jlf_line_format))
            {
                if (jfe.jfe_type != json_log_field::VARIABLE
                    || jfe.jfe_value.pp_value == ts_field
                    || jfe.jfe_value.pp_value == level_field
                    || jfe.jfe_default_value != "-")
                {
                    continue;
                }
                if (!jlu.jlu_format_hits[index]) {
                    jlu.jlu_strikes += 1;
                }
            }
        }
    } else {
        unsigned char* msg;
        int line_count = 2;

        msg = yajl_get_error(
            handle, 1, (const unsigned char*) sbr.get_data(), sbr.length());
        if (msg != nullptr) {
            auto msg_frag = string_fragment::from_c_str(msg);
            log_debug("Unable to parse line at offset %d: %s",
                      li.li_file_range.fr_offset,
                      msg);
            line_count = msg_frag.count('\n') + 1;
            yajl_free_error(handle, msg);
        }
        if (!this->lf_specialized) {
            return scan_no_match{"JSON parsing failed"};
        }
        for (int lpc = 0; lpc < line_count; lpc++) {
            log_level_t level = LEVEL_INVALID;

            ll.set_time(dst.back().get_timeval());
            if (lpc > 0) {
                level = (log_level_t) (level | LEVEL_CONTINUED);
            }
            ll.set_level(level);
            ll.set_sub_offset(lpc);
            dst.emplace_back(ll);
        }
    }

    if (jlu.jlu_quality > 0) {
        jlu.jlu_quality += 3000;
    }
    return scan_match{jlu.jlu_quality, jlu.jlu_strikes};
}

log_format::scan_result_t
external_log_format::scan(logfile& lf,
                          std::vector<logline>& dst,
                          const line_info& li,
                          shared_buffer_ref& sbr,
                          scan_batch_context& sbc)
{
    if (dst.empty()) {
        auto file_options = lf.get_file_options();

        if (file_options) {
            this->lf_date_time.dts_default_zone
                = file_options->second.fo_default_zone.pp_value;
        } else {
            this->lf_date_time.dts_default_zone = nullptr;
        }
    }

    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
        return this->scan_json(dst, li, sbr, sbc);
    }

    int curr_fmt = -1, orig_lock = this->last_pattern_index();
    int pat_index = orig_lock;
    auto line_sf = sbr.to_string_fragment();
    thread_local auto md = lnav::pcre2pp::match_data::unitialized();

    while (::next_format(this->elf_pattern_order, curr_fmt, pat_index)) {
        auto* fpat = this->elf_pattern_order[curr_fmt].get();
        auto* pat = fpat->p_pcre.pp_value.get();

        if (fpat->p_module_format) {
            continue;
        }

        auto found_match
            = pat->capture_from(line_sf).into(md).found_p(PCRE2_NO_UTF_CHECK);
        if (!found_match) {
            if (!this->lf_pattern_locks.empty() && pat_index != -1) {
                curr_fmt = -1;
                pat_index = -1;
            }
            continue;
        }

        auto ts = md[fpat->p_timestamp_field_index];
        auto level_cap = md[fpat->p_level_field_index];
        auto opid_cap = md[fpat->p_opid_field_index];
        auto body_cap = md[fpat->p_body_field_index];
        const char* last;
        exttm log_time_tm;
        timeval log_tv;
        uint8_t mod_index = 0;
        uint16_t opid = 0;
        char combined_datetime_buf[512];

        if (fpat->p_time_field_index != -1) {
            auto time_cap = md[fpat->p_time_field_index];
            if (ts && time_cap) {
                auto ts_str_len = snprintf(combined_datetime_buf,
                                           sizeof(combined_datetime_buf),
                                           "%.*sT%.*s",
                                           ts->length(),
                                           ts->data(),
                                           time_cap->length(),
                                           time_cap->data());
                ts = string_fragment::from_bytes(combined_datetime_buf,
                                                 ts_str_len);
            }
        }

        auto level = this->convert_level(
            level_cap.value_or(string_fragment::invalid()), &sbc);

        if (!ts) {
            level = log_level_t::LEVEL_INVALID;
        } else if ((last
                    = this->lf_date_time.scan(ts->data(),
                                              ts->length(),
                                              this->get_timestamp_formats(),
                                              &log_time_tm,
                                              log_tv))
                   == nullptr)
        {
            auto ls = this->lf_date_time.unlock();
            if ((last = this->lf_date_time.scan(ts->data(),
                                                ts->length(),
                                                this->get_timestamp_formats(),
                                                &log_time_tm,
                                                log_tv))
                == nullptr)
            {
                this->lf_date_time.relock(ls);
                continue;
            }
            if (last != nullptr) {
                auto old_flags = this->lf_timestamp_flags & DATE_TIME_SET_FLAGS;
                auto new_flags = log_time_tm.et_flags & DATE_TIME_SET_FLAGS;

                // It is unlikely a valid timestamp would lose much
                // precision.
                if (new_flags != old_flags) {
                    continue;
                }
            }

            log_debug("%s:%d: date-time re-locked to %d",
                      lf.get_unique_path().c_str(),
                      dst.size(),
                      this->lf_date_time.dts_fmt_lock);
        }

        this->lf_timestamp_flags = log_time_tm.et_flags;

        if (!(this->lf_timestamp_flags
              & (ETF_MILLIS_SET | ETF_MICROS_SET | ETF_NANOS_SET))
            && !dst.empty()
            && dst.back().get_time<std::chrono::seconds>().count()
                == log_tv.tv_sec
            && dst.back()
                    .get_subsecond_time<std::chrono::milliseconds>()
                    .count()
                != 0)
        {
            auto log_ms
                = dst.back().get_subsecond_time<std::chrono::microseconds>();

            log_time_tm.et_nsec
                = std::chrono::duration_cast<std::chrono::nanoseconds>(log_ms)
                      .count();
            log_tv.tv_usec
                = std::chrono::duration_cast<std::chrono::microseconds>(log_ms)
                      .count();
        }

        if (!((log_time_tm.et_flags & ETF_DAY_SET)
              && (log_time_tm.et_flags & ETF_MONTH_SET)
              && (log_time_tm.et_flags & ETF_YEAR_SET)))
        {
            this->check_for_new_year(dst, log_time_tm, log_tv);
        }

        if (opid_cap && !opid_cap->empty()) {
            auto opid_iter = sbc.sbc_opids.insert_op(
                sbc.sbc_allocator, opid_cap.value(), log_tv);
            auto& otr = opid_iter->second;

            otr.otr_level_stats.update_msg_count(level);
            if (fpat->p_subid_field_index != -1) {
                auto subid_cap = md[fpat->p_subid_field_index];
                if (subid_cap && !subid_cap->empty()) {
                    auto* ostr = sbc.sbc_opids.sub_op_in_use(sbc.sbc_allocator,
                                                             opid_iter,
                                                             subid_cap.value(),
                                                             log_tv,
                                                             level);
                    if (ostr != nullptr && ostr->ostr_description.empty()) {
                        log_op_description sub_desc;
                        this->update_op_description(
                            *this->lf_subid_description_def,
                            sub_desc,
                            fpat,
                            md);
                        if (!sub_desc.lod_elements.empty()) {
                            auto& sub_desc_def
                                = this->lf_subid_description_def->at(
                                    sub_desc.lod_id.value());
                            ostr->ostr_description
                                = sub_desc_def.to_string(sub_desc.lod_elements);
                        }
                    }
                }
            }
            this->update_op_description(
                *this->lf_opid_description_def, otr.otr_description, fpat, md);
            opid = opid_cap->hash();
        }
        if (fpat->p_module_field_index != -1) {
            auto mod_cap = md[fpat->p_module_field_index];
            if (mod_cap && body_cap) {
                intern_string_t mod_name
                    = intern_string::lookup(mod_cap.value());
                auto mod_iter = MODULE_FORMATS.find(mod_name);

                if (mod_iter == MODULE_FORMATS.end()) {
                    mod_index = this->module_scan(body_cap.value(), mod_name);
                    mod_iter = MODULE_FORMATS.find(mod_name);
                } else if (mod_iter->second.mf_mod_format) {
                    mod_index = mod_iter->second.mf_mod_format->lf_mod_index;
                }

                if (mod_index && level_cap && body_cap) {
                    auto mod_elf
                        = std::dynamic_pointer_cast<external_log_format>(
                            mod_iter->second.mf_mod_format);

                    if (mod_elf) {
                        thread_local auto mod_md
                            = lnav::pcre2pp::match_data::unitialized();

                        shared_buffer_ref body_ref;

                        body_cap->trim();

                        int mod_pat_index = mod_elf->last_pattern_index();
                        auto& mod_pat
                            = *mod_elf->elf_pattern_order[mod_pat_index];
                        auto match_res = mod_pat.p_pcre.pp_value
                                             ->capture_from(body_cap.value())
                                             .into(mod_md)
                                             .matches(PCRE2_NO_UTF_CHECK)
                                             .ignore_error();
                        if (match_res) {
                            auto mod_level_cap
                                = mod_md[mod_pat.p_level_field_index];

                            level = mod_elf->convert_level(
                                mod_level_cap.value_or(
                                    string_fragment::invalid()),
                                &sbc);
                        }
                    }
                }
            }
        }

        for (const auto& ivd : fpat->p_value_by_index) {
            if (!ivd.ivd_value_def->vd_meta.lvm_values_index) {
                continue;
            }

            ssize_t cap_size = md.capture_size(ivd.ivd_index);
            auto& lvs = this->lf_value_stats[ivd.ivd_value_def->vd_meta
                                                 .lvm_values_index.value()];

            if (cap_size > lvs.lvs_width) {
                lvs.lvs_width = cap_size;
            }
        }

        for (auto value_index : fpat->p_numeric_value_indexes) {
            const indexed_value_def& ivd = fpat->p_value_by_index[value_index];
            const value_def& vd = *ivd.ivd_value_def;
            auto num_cap = md[ivd.ivd_index];

            if (num_cap && num_cap->is_valid()) {
                const struct scaling_factor* scaling = nullptr;

                if (ivd.ivd_unit_field_index >= 0) {
                    auto unit_cap = md[ivd.ivd_unit_field_index];

                    if (unit_cap && unit_cap->is_valid()) {
                        intern_string_t unit_val
                            = intern_string::lookup(unit_cap.value());

                        auto unit_iter = vd.vd_unit_scaling.find(unit_val);
                        if (unit_iter != vd.vd_unit_scaling.end()) {
                            const auto& sf = unit_iter->second;

                            scaling = &sf;
                        }
                    }
                }

                std::optional<double> dvalue_opt;
                switch (vd.vd_meta.lvm_kind) {
                    case value_kind_t::VALUE_INTEGER: {
                        auto scan_res
                            = scn::scan_int<int64_t>(num_cap->to_string_view());
                        if (scan_res) {
                            dvalue_opt = scan_res->value();
                        }
                        break;
                    }
                    case value_kind_t::VALUE_FLOAT: {
                        auto scan_res = scn::scan_value<double>(
                            num_cap->to_string_view());
                        if (scan_res) {
                            dvalue_opt = scan_res->value();
                        }
                        break;
                    }
                    default:
                        break;
                }
                if (dvalue_opt) {
                    auto dvalue = dvalue_opt.value();
                    if (scaling != nullptr) {
                        scaling->scale(dvalue);
                    }
                    this->lf_value_stats[vd.vd_meta.lvm_values_index.value()]
                        .add_value(dvalue);
                }
            }
        }

        dst.emplace_back(
            li.li_file_range.fr_offset, log_tv, level, mod_index, opid);

        if (orig_lock != curr_fmt) {
            uint32_t lock_line;

            log_debug("%s:%zu: changing pattern lock %d -> (%d)%s",
                      lf.get_unique_path().c_str(),
                      dst.size() - 1,
                      orig_lock,
                      curr_fmt,
                      this->elf_pattern_order[curr_fmt]->p_name.c_str());
            if (this->lf_pattern_locks.empty()) {
                lock_line = 0;
            } else {
                lock_line = dst.size() - 1;
            }
            this->lf_pattern_locks.emplace_back(lock_line, curr_fmt);
        }
        return scan_match{1000};
    }

    if (this->lf_specialized && !this->lf_multiline) {
        const auto& last_line = dst.back();

        log_debug("%s: invalid line %d file_offset=%" PRIu64,
                  lf.get_filename().c_str(),
                  dst.size(),
                  li.li_file_range.fr_offset);
        dst.emplace_back(li.li_file_range.fr_offset,
                         last_line.get_timeval(),
                         log_level_t::LEVEL_INVALID);

        return scan_match{0};
    }

    return scan_no_match{"no patterns matched"};
}

uint8_t
external_log_format::module_scan(string_fragment body_cap,
                                 const intern_string_t& mod_name)
{
    uint8_t mod_index;
    body_cap = body_cap.trim();
    auto& ext_fmts = GRAPH_ORDERED_FORMATS;
    module_format mf;

    for (auto& elf : ext_fmts) {
        int curr_fmt = -1, fmt_lock = -1;

        while (::next_format(elf->elf_pattern_order, curr_fmt, fmt_lock)) {
            thread_local auto md = lnav::pcre2pp::match_data::unitialized();

            auto& fpat = elf->elf_pattern_order[curr_fmt];
            auto& pat = fpat->p_pcre;

            if (!fpat->p_module_format) {
                continue;
            }

            auto match_res = pat.pp_value->capture_from(body_cap)
                                 .into(md)
                                 .matches(PCRE2_NO_UTF_CHECK)
                                 .ignore_error();
            if (!match_res) {
                continue;
            }

            log_debug("%s:module format found -- %s (%d)",
                      mod_name.get(),
                      elf->get_name().get(),
                      elf->lf_mod_index);

            mod_index = elf->lf_mod_index;
            mf.mf_mod_format = elf->specialized(curr_fmt);
            MODULE_FORMATS[mod_name] = mf;

            return mod_index;
        }
    }

    MODULE_FORMATS[mod_name] = mf;

    return 0;
}

void
external_log_format::annotate(logfile* lf,
                              uint64_t line_number,
                              string_attrs_t& sa,
                              logline_value_vector& values,
                              bool annotate_module) const
{
    thread_local auto md = lnav::pcre2pp::match_data::unitialized();

    auto& line = values.lvv_sbr;
    line_range lr;

    line.erase_ansi();
    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
        if (this->jlf_cached_opts.full_message) {
            values = this->jlf_line_values;
            sa = this->jlf_line_attrs;
        } else {
            values.lvv_sbr = this->jlf_line_values.lvv_sbr.clone();
            for (const auto& llv : this->jlf_line_values.lvv_values) {
                if (this->jlf_cached_sub_range.contains(llv.lv_origin)) {
                    values.lvv_values.emplace_back(llv);
                    values.lvv_values.back().lv_origin.shift(
                        this->jlf_cached_sub_range.lr_start,
                        -this->jlf_cached_sub_range.lr_start);
                }
            }
            for (const auto& attr : this->jlf_line_attrs) {
                if (this->jlf_cached_sub_range.contains(attr.sa_range)) {
                    sa.emplace_back(attr);
                    sa.back().sa_range.shift(
                        this->jlf_cached_sub_range.lr_start,
                        -this->jlf_cached_sub_range.lr_start);
                }
            }
            values.lvv_opid_value = this->jlf_line_values.lvv_opid_value;
            values.lvv_opid_provenance
                = this->jlf_line_values.lvv_opid_provenance;
        }
        log_format::annotate(lf, line_number, sa, values, annotate_module);
        return;
    }

    if (line.empty()) {
        return;
    }

    values.lvv_values.reserve(this->elf_value_defs.size());

    int pat_index = this->pattern_index_for_line(line_number);
    const auto& pat = *this->elf_pattern_order[pat_index];

    sa.reserve(pat.p_pcre.pp_value->get_capture_count());
    auto match_res
        = pat.p_pcre.pp_value->capture_from(line.to_string_fragment())
              .into(md)
              .matches(PCRE2_NO_UTF_CHECK)
              .ignore_error();
    if (!match_res) {
        // A continued line still needs a body.
        lr.lr_start = 0;
        lr.lr_end = line.length();
        sa.emplace_back(lr, SA_BODY.value());
        if (!this->lf_multiline) {
            auto len
                = pat.p_pcre.pp_value->match_partial(line.to_string_fragment());
            sa.emplace_back(
                line_range{(int) len, -1},
                SA_INVALID.value("Log line does not match any pattern"));
        }
        return;
    }

    std::optional<string_fragment> module_cap;
    if (!pat.p_module_format) {
        auto ts_cap = md[pat.p_timestamp_field_index];
        if (ts_cap) {
            sa.emplace_back(to_line_range(ts_cap.value()), L_TIMESTAMP.value());
        }

        if (pat.p_module_field_index != -1) {
            module_cap = md[pat.p_module_field_index];
            if (module_cap) {
                sa.emplace_back(to_line_range(module_cap.value()),
                                L_MODULE.value());
            }
        }

        auto opid_cap = md[pat.p_opid_field_index];
        if (opid_cap && !opid_cap->empty()) {
            sa.emplace_back(to_line_range(opid_cap.value()), L_OPID.value());
            values.lvv_opid_value = opid_cap->to_string();
            values.lvv_opid_provenance
                = logline_value_vector::opid_provenance::file;
        }
    }

    auto body_cap = md[pat.p_body_field_index];
    auto level_cap = md[pat.p_level_field_index];
    auto src_file_cap = md[pat.p_src_file_field_index];
    auto src_line_cap = md[pat.p_src_line_field_index];

    if (level_cap
        && (!body_cap
            || (body_cap && level_cap->sf_begin != body_cap->sf_begin)))
    {
        sa.emplace_back(to_line_range(level_cap.value()), L_LEVEL.value());
    }

    if (src_file_cap) {
        sa.emplace_back(to_line_range(src_file_cap.value()),
                        SA_SRC_FILE.value());
    }
    if (src_line_cap) {
        sa.emplace_back(to_line_range(src_line_cap.value()),
                        SA_SRC_LINE.value());
    }

    for (size_t lpc = 0; lpc < pat.p_value_by_index.size(); lpc++) {
        const auto& ivd = pat.p_value_by_index[lpc];
        const scaling_factor* scaling = nullptr;
        auto cap = md[ivd.ivd_index];
        const auto& vd = *ivd.ivd_value_def;

        if (ivd.ivd_unit_field_index >= 0) {
            auto unit_cap = md[ivd.ivd_unit_field_index];

            if (unit_cap) {
                intern_string_t unit_val
                    = intern_string::lookup(unit_cap.value());
                auto unit_iter = vd.vd_unit_scaling.find(unit_val);
                if (unit_iter != vd.vd_unit_scaling.end()) {
                    const auto& sf = unit_iter->second;

                    scaling = &sf;
                }
            }
        }

        if (cap) {
            values.lvv_values.emplace_back(
                vd.vd_meta, line, to_line_range(cap.value()));
            values.lvv_values.back().apply_scaling(scaling);
        } else {
            values.lvv_values.emplace_back(vd.vd_meta);
        }
        if (pat.p_module_format) {
            values.lvv_values.back().lv_meta.lvm_from_module = true;
        }
    }

    bool did_mod_annotate_body = false;
    if (annotate_module && module_cap && body_cap && body_cap->is_valid()) {
        intern_string_t mod_name = intern_string::lookup(module_cap.value());
        auto mod_iter = MODULE_FORMATS.find(mod_name);

        if (mod_iter != MODULE_FORMATS.end()
            && mod_iter->second.mf_mod_format != nullptr)
        {
            auto& mf = mod_iter->second;
            auto narrow_res
                = line.narrow(body_cap->sf_begin, body_cap->length());
            auto pre_mod_values_size = values.lvv_values.size();
            auto pre_mod_sa_size = sa.size();
            mf.mf_mod_format->annotate(lf, line_number, sa, values, false);
            for (size_t lpc = pre_mod_values_size;
                 lpc < values.lvv_values.size();
                 lpc++)
            {
                values.lvv_values[lpc].lv_origin.shift(0, body_cap->sf_begin);
            }
            for (size_t lpc = pre_mod_sa_size; lpc < sa.size(); lpc++) {
                sa[lpc].sa_range.shift(0, body_cap->sf_begin);
            }
            line.widen(narrow_res);
            did_mod_annotate_body = true;
        }
    }
    if (!did_mod_annotate_body) {
        if (body_cap && body_cap->is_valid()) {
            lr = to_line_range(body_cap.value());
        } else {
            lr.lr_start = line.length();
            lr.lr_end = line.length();
        }
        sa.emplace_back(lr, SA_BODY.value());
    }

    log_format::annotate(lf, line_number, sa, values, annotate_module);
}

void
external_log_format::rewrite(exec_context& ec,
                             shared_buffer_ref& line,
                             string_attrs_t& sa,
                             std::string& value_out)
{
    auto& values = *ec.ec_line_values;

    value_out.assign(line.get_data(), line.length());

    for (auto iter = values.lvv_values.begin(); iter != values.lvv_values.end();
         ++iter)
    {
        if (!iter->lv_origin.is_valid()) {
            log_debug("%d: not rewriting value with invalid origin -- %s",
                      ec.ec_top_line,
                      iter->lv_meta.lvm_name.get());
            continue;
        }

        auto vd_iter = this->elf_value_defs.find(iter->lv_meta.lvm_name);
        if (vd_iter == this->elf_value_defs.end()) {
            log_debug("not rewriting undefined value -- %s",
                      iter->lv_meta.lvm_name.get());
            continue;
        }

        const auto& vd = *vd_iter->second;

        if (vd.vd_rewriter.empty()) {
            continue;
        }

        auto _sg = ec.enter_source(
            vd_iter->second->vd_rewrite_src_name, 1, vd.vd_rewriter);
        std::string field_value;

        auto_mem<FILE> tmpout(fclose);

        tmpout = std::tmpfile();
        if (!tmpout) {
            log_error("unable to create temporary file");
            return;
        }
        fcntl(fileno(tmpout), F_SETFD, FD_CLOEXEC);
        auto fd_copy = auto_fd::dup_of(fileno(tmpout));
        fd_copy.close_on_exec();
        auto ec_out = std::make_pair(tmpout.release(), fclose);
        {
            exec_context::output_guard og(ec, "tmp", ec_out);

            auto exec_res = execute_any(ec, vd.vd_rewriter);
            if (exec_res.isOk()) {
                field_value = exec_res.unwrap();
            } else {
                field_value = exec_res.unwrapErr().to_attr_line().get_string();
            }
        }
        struct stat st;
        fstat(fd_copy.get(), &st);
        if (st.st_size > 0) {
            auto buf = auto_buffer::alloc(st.st_size);

            buf.resize(st.st_size);
            pread(fd_copy.get(), buf.in(), st.st_size, 0);
            field_value = buf.to_string();
        }
        value_out.erase(iter->lv_origin.lr_start, iter->lv_origin.length());

        int32_t shift_amount
            = ((int32_t) field_value.length()) - iter->lv_origin.length();
        auto orig_lr = iter->lv_origin;
        value_out.insert(iter->lv_origin.lr_start, field_value);
        for (auto shift_iter = values.lvv_values.begin();
             shift_iter != values.lvv_values.end();
             ++shift_iter)
        {
            shift_iter->lv_origin.shift_range(orig_lr, shift_amount);
        }
        shift_string_attrs(sa, orig_lr, shift_amount);
    }
}

static int
read_json_field(yajlpp_parse_context* ypc,
                const unsigned char* str,
                size_t len,
                yajl_string_props_t* props)
{
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    timeval tv_out;
    const auto frag = string_fragment::from_bytes(str, len);
    intern_string_t field_name;
    const auto* vd = jlu->get_field_def(ypc);

    if (vd != nullptr) {
        field_name = vd->vd_meta.lvm_name;
    }

    if (field_name.empty()) {
    } else if (jlu->jlu_format->lf_timestamp_field == field_name) {
        const auto* last = jlu->jlu_format->lf_date_time.scan(
            (const char*) str,
            len,
            jlu->jlu_format->get_timestamp_formats(),
            &jlu->jlu_exttm,
            tv_out);
        if (last == nullptr) {
            auto ls = jlu->jlu_format->lf_date_time.unlock();
            if ((last = jlu->jlu_format->lf_date_time.scan(
                     (const char*) str,
                     len,
                     jlu->jlu_format->get_timestamp_formats(),
                     &jlu->jlu_exttm,
                     tv_out))
                == nullptr)
            {
                jlu->jlu_format->lf_date_time.relock(ls);
            }
            if (last != nullptr) {
                auto old_flags
                    = jlu->jlu_format->lf_timestamp_flags & DATE_TIME_SET_FLAGS;
                auto new_flags = jlu->jlu_exttm.et_flags & DATE_TIME_SET_FLAGS;

                // It is unlikely a valid timestamp would lose much
                // precision.
                if (new_flags != old_flags) {
                    last = nullptr;
                }
            }
        }
        if (last != nullptr) {
            jlu->jlu_format->lf_timestamp_flags = jlu->jlu_exttm.et_flags;
            jlu->jlu_base_line->set_time(tv_out);
        }
    } else if (jlu->jlu_format->elf_level_pointer.pp_value != nullptr) {
        if (jlu->jlu_format->elf_level_pointer.pp_value
                ->find_in(field_name.to_string_fragment(), PCRE2_NO_UTF_CHECK)
                .ignore_error()
                .has_value())
        {
            jlu->jlu_base_line->set_level(
                jlu->jlu_format->convert_level(frag, jlu->jlu_batch_context));
        }
    }
    if (jlu->jlu_format->elf_level_field == field_name) {
        jlu->jlu_base_line->set_level(
            jlu->jlu_format->convert_level(frag, jlu->jlu_batch_context));
    }
    if (jlu->jlu_format->elf_opid_field == field_name) {
        jlu->jlu_base_line->set_opid(frag.hash());

        auto& sbc = *jlu->jlu_batch_context;
        auto opid_iter = sbc.sbc_opids.los_opid_ranges.find(frag);
        if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) {
            jlu->jlu_opid_frag = frag.to_owned(sbc.sbc_allocator);
        } else {
            jlu->jlu_opid_frag = opid_iter->first;
        }
    }
    if (!jlu->jlu_format->elf_subid_field.empty()
        && jlu->jlu_format->elf_subid_field == field_name)
    {
        jlu->jlu_subid = frag.to_string();
    }

    if (vd != nullptr && vd->vd_is_desc_field) {
        auto frag_copy = frag.to_owned(jlu->jlu_format->lf_desc_allocator);

        jlu->jlu_format->lf_desc_captures.emplace(field_name, frag_copy);
    }

    jlu->add_sub_lines_for(vd, ypc->is_level(1), std::nullopt, str, len, props);

    return 1;
}

static int
rewrite_json_field(yajlpp_parse_context* ypc,
                   const unsigned char* str,
                   size_t len,
                   yajl_string_props_t* props)
{
    static const intern_string_t body_name = intern_string::lookup("body", -1);
    auto* jlu = (json_log_userdata*) ypc->ypc_userdata;
    intern_string_t field_name;
    const auto* vd = jlu->get_field_def(ypc);

    if (!ypc->is_level(1) && vd == nullptr) {
        return 1;
    }
    if (vd != nullptr) {
        field_name = vd->vd_meta.lvm_name;
    } else {
        field_name = ypc->get_path();
    }

    if (jlu->jlu_format->elf_opid_field == field_name) {
        auto frag = string_fragment::from_bytes(str, len);
        jlu->jlu_format->jlf_line_values.lvv_opid_value = frag.to_string();
        jlu->jlu_format->jlf_line_values.lvv_opid_provenance
            = logline_value_vector::opid_provenance::file;
    }
    if (jlu->jlu_format->lf_timestamp_field == field_name) {
        char time_buf[64];

        // TODO add a timeval kind to logline_value
        if (jlu->jlu_line->is_time_skewed()
            || (jlu->jlu_format->lf_timestamp_flags
                & (ETF_MICROS_SET | ETF_NANOS_SET | ETF_ZONE_SET)))
        {
            timeval tv;

            const auto* last = jlu->jlu_format->lf_date_time.scan(
                (const char*) str,
                len,
                jlu->jlu_format->get_timestamp_formats(),
                &jlu->jlu_exttm,
                tv);
            if (last == nullptr) {
                auto ls = jlu->jlu_format->lf_date_time.unlock();
                if ((last = jlu->jlu_format->lf_date_time.scan(
                         (const char*) str,
                         len,
                         jlu->jlu_format->get_timestamp_formats(),
                         &jlu->jlu_exttm,
                         tv))
                    == nullptr)
                {
                    jlu->jlu_format->lf_date_time.relock(ls);
                }
            }
            if (!jlu->jlu_subline_opts.hash_hack) {
                if (jlu->jlu_exttm.et_flags & ETF_ZONE_SET
                    && jlu->jlu_format->lf_date_time.dts_zoned_to_local)
                {
                    jlu->jlu_exttm.et_flags &= ~ETF_Z_IS_UTC;
                }
                jlu->jlu_exttm.et_gmtoff
                    = jlu->jlu_format->lf_date_time.dts_local_offset_cache;
            }
            jlu->jlu_format->lf_date_time.ftime(
                time_buf,
                sizeof(time_buf),
                jlu->jlu_format->get_timestamp_formats(),
                jlu->jlu_exttm);
        } else {
            sql_strftime(
                time_buf, sizeof(time_buf), jlu->jlu_line->get_timeval(), 'T');
        }
        jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
            jlu->jlu_format->get_value_meta(field_name,
                                            value_kind_t::VALUE_TEXT),
            std::string{time_buf});
    } else if (jlu->jlu_shared_buffer.contains((const char*) str)) {
        auto str_offset = (int) ((const char*) str - jlu->jlu_line_value);
        if (field_name == jlu->jlu_format->elf_body_field) {
            jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
                logline_value_meta(body_name,
                                   value_kind_t::VALUE_TEXT,
                                   logline_value_meta::internal_column{},
                                   jlu->jlu_format),
                string_fragment::from_byte_range(
                    jlu->jlu_shared_buffer.get_data(),
                    str_offset,
                    str_offset + len));
        }

        jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
            jlu->jlu_format->get_value_meta(field_name,
                                            value_kind_t::VALUE_TEXT),
            string_fragment::from_byte_range(jlu->jlu_shared_buffer.get_data(),
                                             str_offset,
                                             str_offset + len));
    } else {
        if (field_name == jlu->jlu_format->elf_body_field) {
            jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
                logline_value_meta(body_name,
                                   value_kind_t::VALUE_TEXT,
                                   logline_value_meta::internal_column{},
                                   jlu->jlu_format),
                std::string{(const char*) str, len});
        }

        jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
            jlu->jlu_format->get_value_meta(ypc, vd, value_kind_t::VALUE_TEXT),
            std::string{(const char*) str, len});
    }

    return 1;
}

void
external_log_format::get_subline(const logline& ll,
                                 shared_buffer_ref& sbr,
                                 subline_options opts)
{
    if (this->elf_type == elf_type_t::ELF_TYPE_TEXT) {
        return;
    }

    if (this->jlf_cached_offset != ll.get_offset()
        || this->jlf_cached_opts != opts)
    {
        auto& ypc = *(this->jlf_parse_context);
        yajl_handle handle = this->jlf_yajl_handle.get();
        json_log_userdata jlu(sbr, nullptr);

        jlu.jlu_subline_opts = opts;

        this->jlf_share_manager.invalidate_refs();
        this->jlf_cached_line.clear();
        this->jlf_line_values.clear();
        this->jlf_line_offsets.clear();
        this->jlf_line_attrs.clear();

        auto line_frag = sbr.to_string_fragment();

        if (!line_frag.startswith("{")) {
            this->jlf_cached_line.resize(line_frag.length());
            memcpy(this->jlf_cached_line.data(),
                   line_frag.data(),
                   line_frag.length());
            this->jlf_line_values.clear();
            sbr.share(this->jlf_share_manager,
                      &this->jlf_cached_line[0],
                      this->jlf_cached_line.size());
            this->jlf_line_values.lvv_sbr = sbr.clone();
            this->jlf_line_attrs.emplace_back(
                line_range{0, -1},
                SA_INVALID.value(fmt::format(
                    FMT_STRING("line at offset {} is not a JSON-line"),
                    ll.get_offset())));
            return;
        }

        yajl_reset(handle);
        ypc.set_static_handler(json_log_rewrite_handlers.jpc_children[0]);
        ypc.ypc_userdata = &jlu;
        ypc.ypc_ignore_unused = true;
        ypc.ypc_alt_callbacks.yajl_start_array = json_array_start;
        ypc.ypc_alt_callbacks.yajl_end_array = json_array_end;
        ypc.ypc_alt_callbacks.yajl_start_map = json_array_start;
        ypc.ypc_alt_callbacks.yajl_end_map = json_array_end;
        jlu.jlu_format = this;
        jlu.jlu_line = &ll;
        jlu.jlu_handle = handle;
        jlu.jlu_line_value = sbr.get_data();
        jlu.jlu_format_hits.resize(this->jlf_line_format.size());

        yajl_status parse_status = yajl_parse(
            handle, (const unsigned char*) sbr.get_data(), sbr.length());
        if (parse_status != yajl_status_ok
            || yajl_complete_parse(handle) != yajl_status_ok)
        {
            unsigned char* msg;
            std::string full_msg;

            msg = yajl_get_error(
                handle, 1, (const unsigned char*) sbr.get_data(), sbr.length());
            if (msg != nullptr) {
                full_msg = fmt::format(
                    FMT_STRING("[offset: {}] {}\n{}"),
                    ll.get_offset(),
                    fmt::string_view{sbr.get_data(), sbr.length()},
                    reinterpret_cast<char*>(msg));
                yajl_free_error(handle, msg);
            }

            this->jlf_cached_line.resize(full_msg.size());
            memcpy(
                this->jlf_cached_line.data(), full_msg.data(), full_msg.size());
            this->jlf_line_values.clear();
            this->jlf_line_attrs.emplace_back(
                line_range{0, -1},
                SA_INVALID.value("JSON line failed to parse"));
        } else {
            std::vector<logline_value>::iterator lv_iter;
            bool used_values[this->jlf_line_values.lvv_values.size()];
            struct line_range lr;

            memset(used_values, 0, sizeof(used_values));
            for (lv_iter = this->jlf_line_values.lvv_values.begin();
                 lv_iter != this->jlf_line_values.lvv_values.end();
                 ++lv_iter)
            {
                lv_iter->lv_meta.lvm_format = this;
            }
            if (jlu.jlu_opid_frag) {
                this->jlf_line_values.lvv_opid_value
                    = jlu.jlu_opid_frag->to_string();
                this->jlf_line_values.lvv_opid_provenance
                    = logline_value_vector::opid_provenance::file;
            }

            int sub_offset = this->jlf_line_format_init_count;
            for (const auto& jfe : this->jlf_line_format) {
                static const intern_string_t ts_field
                    = intern_string::lookup("__timestamp__", -1);
                static const intern_string_t level_field
                    = intern_string::lookup("__level__");
                size_t begin_size = this->jlf_cached_line.size();

                switch (jfe.jfe_type) {
                    case json_log_field::CONSTANT:
                        this->json_append_to_cache(
                            jfe.jfe_default_value.c_str(),
                            jfe.jfe_default_value.size());
                        break;
                    case json_log_field::VARIABLE:
                        lv_iter = find_if(
                            this->jlf_line_values.lvv_values.begin(),
                            this->jlf_line_values.lvv_values.end(),
                            logline_value_name_cmp(&jfe.jfe_value.pp_value));
                        if (lv_iter != this->jlf_line_values.lvv_values.end()) {
                            auto str = lv_iter->to_string();
                            value_def* vd = nullptr;

                            if (lv_iter->lv_meta.lvm_values_index) {
                                vd = this->elf_value_def_order
                                         [lv_iter->lv_meta.lvm_values_index
                                              .value()]
                                             .get();
                            }
                            while (endswith(str, "\n")) {
                                str.pop_back();
                            }
                            size_t nl_pos = str.find('\n');

                            if (!jfe.jfe_prefix.empty()) {
                                this->json_append_to_cache(jfe.jfe_prefix);
                            }
                            lr.lr_start = this->jlf_cached_line.size();

                            if ((int) str.size() > jfe.jfe_max_width) {
                                switch (jfe.jfe_overflow) {
                                    case json_format_element::overflow_t::
                                        ABBREV: {
                                        size_t new_size
                                            = abbreviate_str(&str[0],
                                                             str.size(),
                                                             jfe.jfe_max_width);
                                        str.resize(new_size);
                                        this->json_append(jfe, vd, str);
                                        break;
                                    }
                                    case json_format_element::overflow_t::
                                        TRUNCATE: {
                                        this->json_append_to_cache(
                                            str.c_str(), jfe.jfe_max_width);
                                        break;
                                    }
                                    case json_format_element::overflow_t::
                                        DOTDOT: {
                                        size_t middle
                                            = (jfe.jfe_max_width / 2) - 1;
                                        this->json_append_to_cache(str.c_str(),
                                                                   middle);
                                        this->json_append_to_cache("..", 2);
                                        size_t rest
                                            = (jfe.jfe_max_width - middle - 2);
                                        this->json_append_to_cache(
                                            str.c_str() + str.size() - rest,
                                            rest);
                                        break;
                                    }
                                    case json_format_element::overflow_t::
                                        LASTWORD: {
                                        size_t new_size
                                            = last_word_str(&str[0],
                                                            str.size(),
                                                            jfe.jfe_max_width);
                                        str.resize(new_size);
                                        this->json_append(jfe, vd, str);
                                        break;
                                    }
                                }
                            } else {
                                sub_offset
                                    += std::count(str.begin(), str.end(), '\n');
                                this->json_append(jfe, vd, str);
                            }

                            if (nl_pos == std::string::npos || opts.full_message) {
                                lr.lr_end = this->jlf_cached_line.size();
                            } else {
                                lr.lr_end = lr.lr_start + nl_pos;
                            }

                            if (lv_iter->lv_meta.lvm_name
                                == this->lf_timestamp_field)
                            {
                                this->jlf_line_attrs.emplace_back(
                                    lr, L_TIMESTAMP.value());
                            } else if (lv_iter->lv_meta.lvm_name
                                       == this->elf_body_field)
                            {
                                this->jlf_line_attrs.emplace_back(
                                lr, SA_BODY.value());
                            } else if (lv_iter->lv_meta.lvm_name
                                       == this->elf_src_file_field)
                            {
                                this->jlf_line_attrs.emplace_back(
                                lr, SA_SRC_FILE.value());
                            } else if (lv_iter->lv_meta.lvm_name
                                       == this->elf_src_line_field)
                            {
                                this->jlf_line_attrs.emplace_back(
                                    lr, SA_SRC_LINE.value());
                            } else if (lv_iter->lv_meta.lvm_name
                                       == this->elf_level_field)
                            {
                                this->jlf_line_attrs.emplace_back(
                                    lr, L_LEVEL.value());
                            } else if (lv_iter->lv_meta.lvm_name
                                           == this->elf_opid_field
                                       && !lr.empty())
                            {
                                this->jlf_line_attrs.emplace_back(
                                    lr, L_OPID.value());
                            }
                            lv_iter->lv_origin = lr;
                            lv_iter->lv_sub_offset = sub_offset;
                            used_values[std::distance(
                                this->jlf_line_values.lvv_values.begin(),
                                lv_iter)]
                                = true;

                            if (!jfe.jfe_suffix.empty()) {
                                this->json_append_to_cache(jfe.jfe_suffix);
                            }
                        } else if (jfe.jfe_value.pp_value == ts_field) {
                            struct line_range lr;
                            ssize_t ts_len;
                            char ts[64];
                            struct exttm et;

                            ll.to_exttm(et);
                            et.et_nsec += jlu.jlu_exttm.et_nsec % 1000;
                            et.et_gmtoff = jlu.jlu_exttm.et_gmtoff;
                            et.et_flags |= jlu.jlu_exttm.et_flags;
                            if (!jfe.jfe_prefix.empty()) {
                                this->json_append_to_cache(jfe.jfe_prefix);
                            }
                            if (jfe.jfe_ts_format.empty()) {
                                ts_len = this->lf_date_time.ftime(
                                    ts,
                                    sizeof(ts),
                                    this->get_timestamp_formats(),
                                    et);
                            } else {
                                ts_len = ftime_fmt(ts,
                                                   sizeof(ts),
                                                   jfe.jfe_ts_format.c_str(),
                                                   et);
                            }
                            lr.lr_start = this->jlf_cached_line.size();
                            this->json_append_to_cache(ts, ts_len);
                            lr.lr_end = this->jlf_cached_line.size();
                            this->jlf_line_attrs.emplace_back(
                                lr, L_TIMESTAMP.value());

                            lv_iter = find_if(
                                this->jlf_line_values.lvv_values.begin(),
                                this->jlf_line_values.lvv_values.end(),
                                logline_value_name_cmp(
                                    &this->lf_timestamp_field));
                            if (lv_iter
                                != this->jlf_line_values.lvv_values.end())
                            {
                                used_values[distance(
                                    this->jlf_line_values.lvv_values.begin(),
                                    lv_iter)]
                                    = true;
                            }
                            if (!jfe.jfe_suffix.empty()) {
                                this->json_append_to_cache(jfe.jfe_suffix);
                            }
                        } else if (jfe.jfe_value.pp_value == level_field
                                   || jfe.jfe_value.pp_value
                                       == this->elf_level_field)
                        {
                            auto level_name = ll.get_level_name();
                            lr.lr_start = this->jlf_cached_line.size();
                            this->json_append(jfe, nullptr, level_name);
                            if (jfe.jfe_auto_width) {
                                this->json_append_to_cache(
                                    MAX_LEVEL_NAME_LEN - level_name.length());
                            }
                            lr.lr_end = this->jlf_cached_line.size();
                            this->jlf_line_attrs.emplace_back(lr,
                                                              L_LEVEL.value());
                        } else if (!jfe.jfe_default_value.empty()) {
                            if (!jfe.jfe_prefix.empty()) {
                                this->json_append_to_cache(jfe.jfe_prefix);
                            }
                            this->json_append(
                                jfe, nullptr, jfe.jfe_default_value);
                            if (!jfe.jfe_suffix.empty()) {
                                this->json_append_to_cache(jfe.jfe_suffix);
                            }
                        }

                        switch (jfe.jfe_text_transform) {
                            case external_log_format::json_format_element::
                                transform_t::NONE:
                                break;
                            case external_log_format::json_format_element::
                                transform_t::UPPERCASE:
                                for (size_t cindex = begin_size;
                                     cindex < this->jlf_cached_line.size();
                                     cindex++)
                                {
                                    this->jlf_cached_line[cindex] = toupper(
                                        this->jlf_cached_line[cindex]);
                                }
                                break;
                            case external_log_format::json_format_element::
                                transform_t::LOWERCASE:
                                for (size_t cindex = begin_size;
                                     cindex < this->jlf_cached_line.size();
                                     cindex++)
                                {
                                    this->jlf_cached_line[cindex] = tolower(
                                        this->jlf_cached_line[cindex]);
                                }
                                break;
                            case external_log_format::json_format_element::
                                transform_t::CAPITALIZE:
                                for (size_t cindex = begin_size;
                                     cindex < begin_size + 1;
                                     cindex++)
                                {
                                    this->jlf_cached_line[cindex] = toupper(
                                        this->jlf_cached_line[cindex]);
                                }
                                for (size_t cindex = begin_size + 1;
                                     cindex < this->jlf_cached_line.size();
                                     cindex++)
                                {
                                    this->jlf_cached_line[cindex] = tolower(
                                        this->jlf_cached_line[cindex]);
                                }
                                break;
                        }
                        break;
                }
            }
            this->json_append_to_cache("\n", 1);
            sub_offset += 1;

            for (size_t lpc = 0; lpc < this->jlf_line_values.lvv_values.size();
                 lpc++)
            {
                static const intern_string_t body_name
                    = intern_string::lookup("body", -1);
                auto& lv = this->jlf_line_values.lvv_values[lpc];

                if (lv.lv_meta.is_hidden() || used_values[lpc]
                    || body_name == lv.lv_meta.lvm_name)
                {
                    continue;
                }

                auto str = lv.to_string();
                while (endswith(str, "\n")) {
                    str.pop_back();
                }

                lv.lv_sub_offset = sub_offset;
                lv.lv_origin.lr_start = this->jlf_cached_line.size() + 2
                    + lv.lv_meta.lvm_name.size() + 2;
                auto frag = string_fragment::from_str(str);
                while (true) {
                    auto utf_scan_res = is_utf8(frag, '\n');

                    this->json_append_to_cache("  ", 2);
                    this->json_append_to_cache(
                        lv.lv_meta.lvm_name.to_string_fragment());
                    this->json_append_to_cache(": ", 2);
                    lr.lr_start = this->jlf_cached_line.size();
                    this->json_append_to_cache(utf_scan_res.usr_valid_frag);
                    lr.lr_end = this->jlf_cached_line.size();
                    if (lv.lv_meta.lvm_name == this->elf_body_field) {
                        this->jlf_line_attrs.emplace_back(lr, SA_BODY.value());
                    } else {
                        this->jlf_line_attrs.emplace_back(
                            lr, SA_EXTRA_CONTENT.value());
                    }
                    this->json_append_to_cache("\n", 1);
                    sub_offset += 1;
                    if (utf_scan_res.usr_remaining) {
                        frag = utf_scan_res.usr_remaining.value();
                    } else {
                        break;
                    }
                }
                lv.lv_origin.lr_end = this->jlf_cached_line.size() - 1;
                if (lv.lv_meta.lvm_name == this->elf_opid_field
                    && !lv.lv_origin.empty())
                {
                    this->jlf_line_attrs.emplace_back(lv.lv_origin,
                                                      L_OPID.value());
                }
            }
        }

        this->jlf_line_offsets.push_back(0);
        for (size_t lpc = 0; lpc < this->jlf_cached_line.size(); lpc++) {
            if (this->jlf_cached_line[lpc] == '\n') {
                this->jlf_line_offsets.push_back(lpc + 1);
            }
        }
        this->jlf_line_offsets.push_back(this->jlf_cached_line.size());
        this->jlf_cached_offset = ll.get_offset();
        this->jlf_cached_opts = opts;
    }

    off_t this_off = 0, next_off = 0;

    if (!this->jlf_line_offsets.empty()
        && ll.get_sub_offset() < this->jlf_line_offsets.size())
    {
        require(ll.get_sub_offset() < this->jlf_line_offsets.size());

        this_off = this->jlf_line_offsets[ll.get_sub_offset()];
        if ((ll.get_sub_offset() + 1) < (int) this->jlf_line_offsets.size()) {
            next_off = this->jlf_line_offsets[ll.get_sub_offset() + 1];
        } else {
            next_off = this->jlf_cached_line.size();
        }
        if (next_off > 0 && this->jlf_cached_line[next_off - 1] == '\n'
            && this_off != next_off)
        {
            next_off -= 1;
        }
    }

    if (opts.full_message) {
        sbr.share(this->jlf_share_manager,
                  &this->jlf_cached_line[0],
                  this->jlf_cached_line.size());
    } else {
        sbr.share(this->jlf_share_manager,
                  this->jlf_cached_line.data() + this_off,
                  next_off - this_off);
    }
    sbr.get_metadata().m_valid_utf = ll.is_valid_utf();
    sbr.get_metadata().m_has_ansi = ll.has_ansi();
    this->jlf_cached_sub_range.lr_start = this_off;
    this->jlf_cached_sub_range.lr_end = next_off;
    this->jlf_line_values.lvv_sbr = sbr.clone();
}

struct compiled_header_expr {
    auto_mem<sqlite3_stmt> che_stmt{sqlite3_finalize};
    bool che_enabled{true};
};

struct format_header_expressions : public lnav_config_listener {
    format_header_expressions() : lnav_config_listener(__FILE__) {}

    auto_sqlite3 e_db;
    std::map<intern_string_t, std::map<std::string, compiled_header_expr>>
        e_header_exprs;
};

using safe_format_header_expressions = safe::Safe<format_header_expressions>;

static safe_format_header_expressions format_header_exprs;

std::optional<external_file_format>
detect_mime_type(const std::filesystem::path& filename)
{
    uint8_t buffer[1024];
    size_t buffer_size = 0;

    {
        auto_fd fd;

        if ((fd = lnav::filesystem::openp(filename, O_RDONLY)) == -1) {
            return std::nullopt;
        }

        ssize_t rc;

        if ((rc = read(fd, buffer, sizeof(buffer))) == -1) {
            return std::nullopt;
        }
        buffer_size = rc;
    }

    auto hexbuf = auto_buffer::alloc(buffer_size * 2);

    for (size_t lpc = 0; lpc < buffer_size; lpc++) {
        fmt::format_to(
            std::back_inserter(hexbuf), FMT_STRING("{:02x}"), buffer[lpc]);
    }

    safe::WriteAccess<safe_format_header_expressions> in(format_header_exprs);

    for (const auto& format : log_format::get_root_formats()) {
        auto elf = std::dynamic_pointer_cast<external_log_format>(format);
        if (elf == nullptr) {
            continue;
        }

        if (elf->elf_converter.c_header.h_exprs.he_exprs.empty()) {
            continue;
        }

        if (buffer_size < elf->elf_converter.c_header.h_size) {
            log_debug(
                "%s: file content too small (%d) for header detection: %s",
                filename.c_str(),
                buffer_size,
                elf->get_name().get());
            continue;
        }
        for (const auto& hpair : elf->elf_converter.c_header.h_exprs.he_exprs) {
            auto& he = in->e_header_exprs[elf->get_name()][hpair.first];

            if (!he.che_enabled) {
                continue;
            }

            auto* stmt = he.che_stmt.in();

            if (stmt == nullptr) {
                continue;
            }
            sqlite3_reset(stmt);
            auto count = sqlite3_bind_parameter_count(stmt);
            for (int lpc = 0; lpc < count; lpc++) {
                const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);

                if (name[0] == '$') {
                    const char* env_value;

                    if ((env_value = getenv(&name[1])) != nullptr) {
                        sqlite3_bind_text(
                            stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
                    }
                    continue;
                }
                if (strcmp(name, ":header") == 0) {
                    sqlite3_bind_text(stmt,
                                      lpc + 1,
                                      hexbuf.in(),
                                      hexbuf.size(),
                                      SQLITE_STATIC);
                    continue;
                }
                if (strcmp(name, ":filepath") == 0) {
                    sqlite3_bind_text(
                        stmt, lpc + 1, filename.c_str(), -1, SQLITE_STATIC);
                    continue;
                }
            }

            auto step_res = sqlite3_step(stmt);

            switch (step_res) {
                case SQLITE_OK:
                case SQLITE_DONE:
                    continue;
                case SQLITE_ROW:
                    break;
                default: {
                    log_error(
                        "failed to execute file-format header expression: "
                        "%s:%s -- %s",
                        elf->get_name().get(),
                        hpair.first.c_str(),
                        sqlite3_errmsg(in->e_db));
                    he.che_enabled = false;
                    continue;
                }
            }

            log_info("detected format for: %s -- %s (header-expr: %s)",
                     filename.c_str(),
                     elf->get_name().get(),
                     hpair.first.c_str());
            return external_file_format{
                elf->get_name().to_string(),
                elf->elf_converter.c_command.pp_value,
                elf->elf_converter.c_command.pp_location.sl_source.to_string(),
            };
        }
    }

    return std::nullopt;
}

log_format::scan_result_t
log_format::test_line(sample_t& sample,
                      std::vector<lnav::console::user_message>& msgs)
{
    return scan_no_match{};
}

log_format::scan_result_t
external_log_format::test_line(sample_t& sample,
                               std::vector<lnav::console::user_message>& msgs)
{
    auto lines
        = string_fragment::from_str(sample.s_line.pp_value).split_lines();

    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
        auto alloc = ArenaAlloc::Alloc<char>{};
        auto sbc = scan_batch_context{
            alloc,
        };
        std::vector<logline> dst;
        auto li = line_info{
            {0, lines[0].length()},
        };
        shared_buffer sb;
        shared_buffer_ref sbr;
        sbr.share(sb, lines[0].data(), (size_t) lines[0].length());

        return this->scan_json(dst, li, sbr, sbc);
    }

    scan_result_t retval = scan_no_match{"no patterns matched"};
    auto found = false;

    for (auto pat_iter = this->elf_pattern_order.begin();
         pat_iter != this->elf_pattern_order.end();
         ++pat_iter)
    {
        auto& pat = *(*pat_iter);

        if (!pat.p_pcre.pp_value) {
            continue;
        }

        auto md = pat.p_pcre.pp_value->create_match_data();
        auto match_res = pat.p_pcre.pp_value->capture_from(lines[0])
                             .into(md)
                             .matches(PCRE2_NO_UTF_CHECK)
                             .ignore_error();
        if (!match_res) {
            continue;
        }
        retval = scan_match{1000};
        found = true;

        if (pat.p_module_format) {
            continue;
        }

        sample.s_matched_regexes.insert(pat.p_name.to_string());

        const auto ts_cap = md[pat.p_timestamp_field_index];
        const auto level_cap = md[pat.p_level_field_index];
        const char* const* custom_formats = this->get_timestamp_formats();
        date_time_scanner dts;
        timeval tv;
        exttm tm;

        if (ts_cap && ts_cap->sf_begin == 0) {
            pat.p_timestamp_end = ts_cap->sf_end;
        }
        const char* dts_scan_res = nullptr;

        if (ts_cap) {
            dts_scan_res = dts.scan(
                ts_cap->data(), ts_cap->length(), custom_formats, &tm, tv);
        }
        if (dts_scan_res != nullptr) {
            if (dts_scan_res != ts_cap->data() + ts_cap->length()) {
                auto match_len = dts_scan_res - ts_cap->data();
                auto notes = attr_line_t("the used timestamp format: ");
                if (custom_formats == nullptr) {
                    notes.append(PTIMEC_FORMATS[dts.dts_fmt_lock].pf_fmt);
                } else {
                    notes.append(custom_formats[dts.dts_fmt_lock]);
                }
                notes.append("\n  ")
                    .append(ts_cap.value())
                    .append("\n")
                    .append(2 + match_len, ' ')
                    .append("^ matched up to here"_snippet_border);
                auto um = lnav::console::user_message::warning(
                              attr_line_t("timestamp was not fully matched: ")
                                  .append_quoted(ts_cap.value()))
                              .with_snippet(sample.s_line.to_snippet())
                              .with_note(notes)
                              .move();

                msgs.emplace_back(um);
            }
        } else if (!ts_cap) {
            msgs.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t("invalid sample log message: ")
                        .append(lnav::to_json(sample.s_line.pp_value)))
                    .with_reason(attr_line_t("timestamp was not captured"))
                    .with_snippet(sample.s_line.to_snippet())
                    .with_help(attr_line_t(
                        "A timestamp needs to be captured in order for a "
                        "line to be recognized as a log message")));
        } else {
            attr_line_t notes;

            if (custom_formats == nullptr) {
                notes.append("the following built-in formats were tried:");
                for (int lpc = 0; PTIMEC_FORMATS[lpc].pf_fmt != nullptr; lpc++)
                {
                    off_t off = 0;

                    PTIMEC_FORMATS[lpc].pf_func(
                        &tm, ts_cap->data(), off, ts_cap->length());
                    notes.append("\n  ")
                        .append(ts_cap.value())
                        .append("\n")
                        .append(2 + off, ' ')
                        .append("^ "_snippet_border)
                        .append_quoted(
                            lnav::roles::symbol(PTIMEC_FORMATS[lpc].pf_fmt))
                        .append(" matched up to here"_snippet_border);
                }
            } else {
                notes.append("the following custom formats were tried:");
                for (int lpc = 0; custom_formats[lpc] != nullptr; lpc++) {
                    off_t off = 0;

                    ptime_fmt(custom_formats[lpc],
                              &tm,
                              ts_cap->data(),
                              off,
                              ts_cap->length());
                    notes.append("\n  ")
                        .append(ts_cap.value())
                        .append("\n")
                        .append(2 + off, ' ')
                        .append("^ "_snippet_border)
                        .append_quoted(lnav::roles::symbol(custom_formats[lpc]))
                        .append(" matched up to here"_snippet_border);
                }
            }

            msgs.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t("invalid sample log message: ")
                        .append(lnav::to_json(sample.s_line.pp_value)))
                    .with_reason(attr_line_t("unrecognized timestamp -- ")
                                     .append(ts_cap.value()))
                    .with_snippet(sample.s_line.to_snippet())
                    .with_note(notes)
                    .with_help(attr_line_t("If the timestamp format is not "
                                           "supported by default, you can "
                                           "add a custom format with the ")
                                   .append_quoted("timestamp-format"_symbol)
                                   .append(" property")));
        }

        auto level = this->convert_level(
            level_cap.value_or(string_fragment::invalid()), nullptr);

        if (sample.s_level != LEVEL_UNKNOWN && sample.s_level != level) {
            attr_line_t note_al;

            note_al.append("matched regex = ")
                .append(lnav::roles::symbol(pat.p_name.to_string()))
                .append("\n")
                .append("captured level = ")
                .append_quoted(level_cap->to_string());
            if (level_cap && !this->elf_level_patterns.empty()) {
                thread_local auto md = lnav::pcre2pp::match_data::unitialized();

                note_al.append("\nlevel regular expression match results:");
                for (const auto& level_pattern : this->elf_level_patterns) {
                    attr_line_t regex_al
                        = level_pattern.second.lp_pcre.pp_value->get_pattern();
                    lnav::snippets::regex_highlighter(
                        regex_al, -1, line_range{0, (int) regex_al.length()});
                    note_al.append("\n  ")
                        .append(lnav::roles::symbol(
                            level_pattern.second.lp_pcre.pp_path.to_string()))
                        .append(" = ")
                        .append(regex_al)
                        .append("\n    ");
                    auto match_res = level_pattern.second.lp_pcre.pp_value
                                         ->capture_from(level_cap.value())
                                         .into(md)
                                         .matches(PCRE2_NO_UTF_CHECK)
                                         .ignore_error();
                    if (!match_res) {
                        note_al.append(lnav::roles::warning("no match"));
                        continue;
                    }

                    note_al.append(level_cap.value())
                        .append("\n    ")
                        .append(md.leading().length(), ' ')
                        .append("^"_snippet_border);
                    if (match_res->f_all.length() > 2) {
                        note_al.append(lnav::roles::snippet_border(
                            std::string(match_res->f_all.length() - 2, '-')));
                    }
                    if (match_res->f_all.length() > 1) {
                        note_al.append("^"_snippet_border);
                    }
                }
            }
            auto um
                = lnav::console::user_message::error(
                      attr_line_t("invalid sample log message: ")
                          .append(lnav::to_json(sample.s_line.pp_value)))
                      .with_reason(attr_line_t()
                                       .append_quoted(lnav::roles::symbol(
                                           level_names[level]))
                                       .append(" does not match the expected "
                                               "level of ")
                                       .append_quoted(lnav::roles::symbol(
                                           level_names[sample.s_level])))
                      .with_snippet(sample.s_line.to_snippet())
                      .with_note(note_al)
                      .move();
            if (!this->elf_level_patterns.empty()) {
                um.with_help(
                    attr_line_t("Level regexes are not anchored to the "
                                "start/end of the string.  Prepend ")
                        .append_quoted("^"_symbol)
                        .append(" to the expression to match from the "
                                "start of the string and append ")
                        .append_quoted("$"_symbol)
                        .append(" to match up to the end of the string."));
            }
            msgs.emplace_back(um);
        }

        {
            auto full_match_res
                = pat.p_pcre.pp_value->capture_from(sample.s_line.pp_value)
                      .into(md)
                      .matches()
                      .ignore_error();
            if (!full_match_res) {
                attr_line_t regex_al = pat.p_pcre.pp_value->get_pattern();
                lnav::snippets::regex_highlighter(
                    regex_al, -1, line_range{0, (int) regex_al.length()});
                msgs.emplace_back(
                    lnav::console::user_message::error(
                        attr_line_t("invalid pattern: ")
                            .append_quoted(
                                lnav::roles::symbol(pat.p_name.to_string())))
                        .with_reason("pattern does not match entire "
                                     "multiline sample message")
                        .with_snippet(sample.s_line.to_snippet())
                        .with_note(attr_line_t()
                                       .append(lnav::roles::symbol(
                                           pat.p_name.to_string()))
                                       .append(" = ")
                                       .append(regex_al))
                        .with_help(
                            attr_line_t("use ").append_quoted(".*").append(
                                " to match new-lines")));
            } else if (static_cast<size_t>(full_match_res->f_all.length())
                       != sample.s_line.pp_value.length())
            {
                attr_line_t regex_al = pat.p_pcre.pp_value->get_pattern();
                lnav::snippets::regex_highlighter(
                    regex_al, -1, line_range{0, (int) regex_al.length()});
                auto match_length
                    = static_cast<size_t>(full_match_res->f_all.length());
                attr_line_t sample_al = sample.s_line.pp_value;
                sample_al.append("\n")
                    .append(match_length, ' ')
                    .append("^ matched up to here"_error)
                    .with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE));
                auto sample_snippet = lnav::console::snippet::from(
                    sample.s_line.pp_location, sample_al);
                msgs.emplace_back(
                    lnav::console::user_message::error(
                        attr_line_t("invalid pattern: ")
                            .append_quoted(
                                lnav::roles::symbol(pat.p_name.to_string())))
                        .with_reason("pattern does not match entire "
                                     "message")
                        .with_snippet(sample_snippet)
                        .with_note(attr_line_t()
                                       .append(lnav::roles::symbol(
                                           pat.p_name.to_string()))
                                       .append(" = ")
                                       .append(regex_al))
                        .with_help("update the regular expression to fully "
                                   "capture the sample message"));
            }
        }
    }

    if (!found && !this->elf_pattern_order.empty()) {
        std::vector<std::pair<ssize_t, intern_string_t>> partial_indexes;
        attr_line_t notes;
        size_t max_name_width = 0;

        for (const auto& pat_iter : this->elf_pattern_order) {
            auto& pat = *pat_iter;

            if (!pat.p_pcre.pp_value) {
                continue;
            }

            partial_indexes.emplace_back(
                pat.p_pcre.pp_value->match_partial(lines[0]), pat.p_name);
            max_name_width = std::max(max_name_width, pat.p_name.size());
        }
        for (const auto& line_frag : lines) {
            auto src_line = attr_line_t(line_frag.to_string());
            if (!line_frag.endswith("\n")) {
                src_line.append("\n");
            }
            src_line.with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE));
            notes.append("   ").append(src_line);
            for (auto& part_pair : partial_indexes) {
                if (part_pair.first >= 0
                    && part_pair.first < line_frag.length())
                {
                    notes.append("   ")
                        .append(part_pair.first, ' ')
                        .append("^ "_snippet_border)
                        .append(
                            lnav::roles::symbol(part_pair.second.to_string()))
                        .append(" matched up to here"_snippet_border)
                        .append("\n");
                }
                part_pair.first -= line_frag.length();
            }
        }
        notes.add_header(
            "the following shows how each pattern matched this sample:\n");

        attr_line_t regex_note;
        for (const auto& pat_iter : this->elf_pattern_order) {
            if (!pat_iter->p_pcre.pp_value) {
                regex_note
                    .append(lnav::roles::symbol(fmt::format(
                        FMT_STRING("{:{}}"), pat_iter->p_name, max_name_width)))
                    .append(" is invalid");
                continue;
            }

            attr_line_t regex_al = pat_iter->p_pcre.pp_value->get_pattern();
            lnav::snippets::regex_highlighter(
                regex_al, -1, line_range{0, (int) regex_al.length()});

            regex_note
                .append(lnav::roles::symbol(fmt::format(
                    FMT_STRING("{:{}}"), pat_iter->p_name, max_name_width)))
                .append(" = ")
                .append_quoted(regex_al)
                .append("\n");
        }

        msgs.emplace_back(
            lnav::console::user_message::error(
                attr_line_t("invalid sample log message: ")
                    .append(lnav::to_json(sample.s_line.pp_value)))
                .with_reason("sample does not match any patterns")
                .with_snippet(sample.s_line.to_snippet())
                .with_note(notes.rtrim())
                .with_note(regex_note));
    }

    return retval;
}

void
external_log_format::build(std::vector<lnav::console::user_message>& errors)
{
    auto& vc = view_colors::singleton();

    if (!this->lf_timestamp_field.empty()) {
        auto& vd = this->elf_value_defs[this->lf_timestamp_field];
        if (vd.get() == nullptr) {
            vd = std::make_shared<value_def>(
                this->lf_timestamp_field,
                value_kind_t::VALUE_TEXT,
                logline_value_meta::internal_column{},
                this);
            if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
                this->elf_value_def_order.emplace_back(vd);
            }
        }
        vd->vd_meta.lvm_name = this->lf_timestamp_field;
        vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
        vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
        vd->vd_internal = true;

        this->elf_value_defs[LOG_TIME_STR] = vd;
    }

    if (!this->lf_subsecond_field.empty()) {
        if (!this->lf_subsecond_unit.has_value()) {
            errors.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t()
                        .append_quoted(
                            lnav::roles::symbol(this->elf_name.to_string()))
                        .append(" is not a valid log format"))
                    .with_reason(attr_line_t()
                                     .append_quoted("subsecond-units"_symbol)
                                     .append(" must be set when ")
                                     .append_quoted("subsecond-field"_symbol)
                                     .append(" is used"))
                    .with_snippets(this->get_snippets()));
        } else {
            auto& vd = this->elf_value_defs[this->lf_subsecond_field];
            if (vd.get() == nullptr) {
                vd = std::make_shared<value_def>(
                    this->lf_subsecond_field,
                    value_kind_t::VALUE_INTEGER,
                    logline_value_meta::internal_column{},
                    this);
                if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
                    this->elf_value_def_order.emplace_back(vd);
                }
            }
            vd->vd_meta.lvm_name = this->lf_subsecond_field;
            vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
            vd->vd_meta.lvm_hidden = true;
            vd->vd_internal = true;
        }
    }

    if (startswith(this->elf_level_field.get(), "/")) {
        this->elf_level_field
            = intern_string::lookup(this->elf_level_field.get() + 1);
    }
    if (!this->elf_level_field.empty()) {
        auto level_iter = this->elf_value_defs.find(this->elf_level_field);
        if (level_iter == this->elf_value_defs.end()) {
            auto& vd = this->elf_value_defs[this->elf_level_field];
            vd = std::make_shared<value_def>(
                this->elf_level_field,
                value_kind_t::VALUE_TEXT,
                logline_value_meta::internal_column{},
                this);
            if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
                this->elf_value_def_order.emplace_back(vd);
            }
            vd->vd_meta.lvm_name = this->elf_level_field;
            vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
            vd->vd_internal = true;

            if (this->elf_level_field != this->elf_body_field) {
                this->elf_value_defs[LOG_LEVEL_STR] = vd;
            }
        } else {
            if (level_iter->second->vd_meta.lvm_kind
                != value_kind_t::VALUE_TEXT)
            {
                this->lf_level_hideable = false;
            }
            this->elf_value_defs[LOG_LEVEL_STR] = level_iter->second;
        }
    }

    auto opid_field_iter = this->elf_value_defs.find(LOG_OPID_STR);
    if (opid_field_iter == this->elf_value_defs.end()) {
        auto vd
            = std::make_shared<value_def>(this->elf_opid_field,
                                          value_kind_t::VALUE_TEXT,
                                          logline_value_meta::internal_column{},
                                          this);
        if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
            this->elf_value_def_order.emplace_back(vd);
        }
        vd->vd_meta.lvm_name = LOG_OPID_STR;
        vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
        vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
        vd->vd_internal = true;

        this->elf_value_defs[LOG_OPID_STR] = vd;
    }

    if (!this->elf_body_field.empty()) {
        auto& vd = this->elf_value_defs[this->elf_body_field];
        if (vd.get() == nullptr) {
            vd = std::make_shared<value_def>(
                this->elf_body_field,
                value_kind_t::VALUE_TEXT,
                logline_value_meta::internal_column{},
                this);
            if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
                this->elf_value_def_order.emplace_back(vd);
            }
        }
        vd->vd_meta.lvm_name = this->elf_body_field;
        vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
        vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
        vd->vd_internal = true;
    }

    if (!this->elf_src_file_field.empty()) {
        auto& vd = this->elf_value_defs[this->elf_src_file_field];
        if (vd.get() == nullptr) {
            vd = std::make_shared<value_def>(
                this->elf_src_file_field,
                value_kind_t::VALUE_TEXT,
                logline_value_meta::internal_column{},
                this);
        }
        vd->vd_meta.lvm_name = this->elf_src_file_field;
        vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
        vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
    }

    if (!this->elf_src_line_field.empty()) {
        auto& vd = this->elf_value_defs[this->elf_src_line_field];
        if (vd.get() == nullptr) {
            vd = std::make_shared<value_def>(
                this->elf_src_line_field,
                value_kind_t::VALUE_INTEGER,
                logline_value_meta::internal_column{},
                this);
        }
        vd->vd_meta.lvm_name = this->elf_src_line_field;
        vd->vd_meta.lvm_kind = value_kind_t::VALUE_INTEGER;
        vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
    }

    if (!this->lf_timestamp_format.empty()) {
        this->lf_timestamp_format.push_back(nullptr);
    }
    for (auto iter = this->elf_patterns.begin();
         iter != this->elf_patterns.end();
         ++iter)
    {
        pattern& pat = *iter->second;

        if (pat.p_pcre.pp_value == nullptr) {
            continue;
        }

        if (pat.p_module_format) {
            this->elf_has_module_format = true;
        }

        for (auto named_cap : pat.p_pcre.pp_value->get_named_captures()) {
            const intern_string_t name
                = intern_string::lookup(named_cap.get_name());

            if (name == this->lf_timestamp_field) {
                pat.p_timestamp_field_index = named_cap.get_index();
            }
            if (name == this->lf_time_field) {
                pat.p_time_field_index = named_cap.get_index();
            }
            if (name == this->elf_level_field) {
                pat.p_level_field_index = named_cap.get_index();
            }
            if (name == this->elf_module_id_field) {
                pat.p_module_field_index = named_cap.get_index();
            }
            if (name == this->elf_opid_field) {
                pat.p_opid_field_index = named_cap.get_index();
            }
            if (name == this->elf_subid_field) {
                pat.p_subid_field_index = named_cap.get_index();
            }
            if (name == this->elf_body_field) {
                pat.p_body_field_index = named_cap.get_index();
            }
            if (name == this->elf_src_file_field) {
                pat.p_src_file_field_index = named_cap.get_index();
            }
            if (name == this->elf_src_line_field) {
                pat.p_src_line_field_index = named_cap.get_index();
            }

            auto value_iter = this->elf_value_defs.find(name);
            if (value_iter != this->elf_value_defs.end()) {
                auto vd = value_iter->second;
                indexed_value_def ivd;

                ivd.ivd_index = named_cap.get_index();
                if (!vd->vd_unit_field.empty()) {
                    ivd.ivd_unit_field_index = pat.p_pcre.pp_value->name_index(
                        vd->vd_unit_field.get());
                } else {
                    ivd.ivd_unit_field_index = -1;
                }
                if (!vd->vd_internal
                    && !vd->vd_meta.lvm_column
                            .is<logline_value_meta::table_column>())
                {
                    vd->vd_meta.lvm_column = logline_value_meta::table_column{
                        this->elf_column_count++};
                }
                ivd.ivd_value_def = vd;
                pat.p_value_by_index.push_back(ivd);
            }
            pat.p_value_name_to_index[name] = named_cap.get_index();
        }

        stable_sort(pat.p_value_by_index.begin(), pat.p_value_by_index.end());

        for (int lpc = 0; lpc < (int) pat.p_value_by_index.size(); lpc++) {
            auto& ivd = pat.p_value_by_index[lpc];
            auto vd = ivd.ivd_value_def;

            if (!vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier) {
                switch (vd->vd_meta.lvm_kind) {
                    case value_kind_t::VALUE_INTEGER:
                    case value_kind_t::VALUE_FLOAT:
                        pat.p_numeric_value_indexes.push_back(lpc);
                        break;
                    default:
                        break;
                }
            }
        }

        if (!pat.p_module_format && pat.p_timestamp_field_index == -1) {
            errors.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t("invalid pattern: ")
                        .append_quoted(lnav::roles::symbol(pat.p_config_path)))
                    .with_reason("no timestamp capture found in the pattern")
                    .with_snippets(this->get_snippets())
                    .with_help("all log messages need a timestamp"));
        }

        if (!this->elf_level_field.empty() && pat.p_level_field_index == -1) {
            log_warning("%s:level field '%s' not found in pattern",
                        pat.p_config_path.c_str(),
                        this->elf_level_field.get());
        }
        if (!this->elf_module_id_field.empty()
            && pat.p_module_field_index == -1)
        {
            log_warning("%s:module field '%s' not found in pattern",
                        pat.p_config_path.c_str(),
                        this->elf_module_id_field.get());
        }
        if (!this->elf_body_field.empty() && pat.p_body_field_index == -1) {
            log_warning("%s:body field '%s' not found in pattern",
                        pat.p_config_path.c_str(),
                        this->elf_body_field.get());
        }

        this->elf_pattern_order.push_back(iter->second);
    }

    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
        if (!this->elf_patterns.empty()) {
            errors.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t()
                        .append_quoted(
                            lnav::roles::symbol(this->elf_name.to_string()))
                        .append(" is not a valid log format"))
                    .with_reason("structured logs cannot have regexes")
                    .with_snippets(this->get_snippets()));
        }
        if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
            this->lf_multiline = true;
            this->lf_structured = true;
            this->lf_formatted_lines = true;
            this->jlf_parse_context
                = std::make_shared<yajlpp_parse_context>(this->elf_name);
            this->jlf_yajl_handle.reset(
                yajl_alloc(&this->jlf_parse_context->ypc_callbacks,
                           nullptr,
                           this->jlf_parse_context.get()),
                yajl_handle_deleter());
            yajl_config(
                this->jlf_yajl_handle.get(), yajl_dont_validate_strings, 1);
        }
    } else {
        if (this->elf_patterns.empty()) {
            errors.emplace_back(lnav::console::user_message::error(
                                    attr_line_t()
                                        .append_quoted(lnav::roles::symbol(
                                            this->elf_name.to_string()))
                                        .append(" is not a valid log format"))
                                    .with_reason("no regexes specified")
                                    .with_snippets(this->get_snippets()));
        }
    }

    stable_sort(this->elf_level_pairs.begin(), this->elf_level_pairs.end());

    {
        safe::WriteAccess<safe_format_header_expressions> hexprs(
            format_header_exprs);

        if (hexprs->e_db.in() == nullptr) {
            if (sqlite3_open(":memory:", hexprs->e_db.out()) != SQLITE_OK) {
                log_error("unable to open memory DB");
                return;
            }
            register_sqlite_funcs(hexprs->e_db.in(), sqlite_registration_funcs);
        }

        for (const auto& hpair : this->elf_converter.c_header.h_exprs.he_exprs)
        {
            auto stmt_str
                = fmt::format(FMT_STRING("SELECT 1 WHERE {}"), hpair.second);
            compiled_header_expr che;

            log_info("preparing file-format header expression: %s",
                     stmt_str.c_str());
            auto retcode = sqlite3_prepare_v2(hexprs->e_db.in(),
                                              stmt_str.c_str(),
                                              stmt_str.size(),
                                              che.che_stmt.out(),
                                              nullptr);
            if (retcode != SQLITE_OK) {
                auto sql_al = attr_line_t(hpair.second)
                                  .with_attr_for_all(SA_PREFORMATTED.value())
                                  .with_attr_for_all(
                                      VC_ROLE.value(role_t::VCR_QUOTED_CODE))
                                  .move();
                readline_sql_highlighter(
                    sql_al, lnav::sql::dialect::sqlite, std::nullopt);
                intern_string_t watch_expr_path = intern_string::lookup(
                    fmt::format(FMT_STRING("/{}/converter/header/expr/{}"),
                                this->elf_name,
                                hpair.first));
                auto snippet = lnav::console::snippet::from(
                    source_location(watch_expr_path), sql_al);

                auto um = lnav::console::user_message::error(
                              "SQL expression is invalid")
                              .with_reason(sqlite3_errmsg(hexprs->e_db.in()))
                              .with_snippet(snippet)
                              .move();

                errors.emplace_back(um);
                continue;
            }

            hexprs->e_header_exprs[this->elf_name][hpair.first]
                = std::move(che);
        }

        if (!this->elf_converter.c_header.h_exprs.he_exprs.empty()
            && this->elf_converter.c_command.pp_value.empty())
        {
            auto um = lnav::console::user_message::error(
                          "A command is required when a converter is defined")
                          .with_help(
                              "The converter command transforms the file "
                              "into a format that can be consumed by lnav")
                          .with_snippets(this->get_snippets())
                          .move();
            errors.emplace_back(um);
        }
    }

    for (auto& vd : this->elf_value_def_order) {
        std::vector<std::string>::iterator act_iter;

        if (!vd->vd_internal
            && log_vtab_impl::RESERVED_COLUMNS.count(
                vd->vd_meta.lvm_name.to_string_fragment()))
        {
            auto um = lnav::console::user_message::error(
                          attr_line_t("value name ")
                              .append_quoted(lnav::roles::symbol(
                                  fmt::format(FMT_STRING("/{}/value/{}"),
                                              this->elf_name,
                                              vd->vd_meta.lvm_name)))
                              .append(" is reserved and cannot be used"))
                          .with_reason(
                              "lnav automatically defines several columns in "
                              "the log virtual table")
                          .with_snippets(this->get_snippets())
                          .with_help("Choose another name")
                          .move();
            errors.emplace_back(um);
        }

        vd->vd_meta.lvm_format = this;
        if (!vd->vd_internal
            && !vd->vd_meta.lvm_column.is<logline_value_meta::table_column>())
        {
            vd->vd_meta.lvm_column
                = logline_value_meta::table_column{this->elf_column_count++};
        }

        if (vd->vd_meta.lvm_kind == value_kind_t::VALUE_UNKNOWN) {
            vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
        }

        if (this->elf_type == elf_type_t::ELF_TYPE_TEXT) {
            std::set<std::string> available_captures;

            bool found_in_pattern = false;
            for (const auto& pat : this->elf_patterns) {
                if (pat.second->p_pcre.pp_value == nullptr) {
                    continue;
                }

                auto cap_index = pat.second->p_pcre.pp_value->name_index(
                    vd->vd_meta.lvm_name.get());
                if (cap_index >= 0) {
                    found_in_pattern = true;
                    break;
                }

                for (auto named_cap :
                     pat.second->p_pcre.pp_value->get_named_captures())
                {
                    available_captures.insert(named_cap.get_name().to_string());
                }
            }
            if (!found_in_pattern) {
                auto notes
                    = attr_line_t("the following captures are available:\n  ")
                          .join(available_captures,
                                VC_ROLE.value(role_t::VCR_SYMBOL),
                                ", ")
                          .move();
                errors.emplace_back(
                    lnav::console::user_message::warning(
                        attr_line_t("invalid value ")
                            .append_quoted(lnav::roles::symbol(
                                fmt::format(FMT_STRING("/{}/value/{}"),
                                            this->elf_name,
                                            vd->vd_meta.lvm_name.get()))))
                        .with_reason(
                            attr_line_t("no patterns have a capture named ")
                                .append_quoted(vd->vd_meta.lvm_name.get()))
                        .with_note(notes)
                        .with_snippets(this->get_snippets())
                        .with_help("values are populated from captures in "
                                   "patterns, so at least one pattern must "
                                   "have a capture with this value name"));
            }
        }

        for (act_iter = vd->vd_action_list.begin();
             act_iter != vd->vd_action_list.end();
             ++act_iter)
        {
            if (this->lf_action_defs.find(*act_iter)
                == this->lf_action_defs.end())
            {
#if 0
                errors.push_back("error:" + this->elf_name.to_string() + ":"
                                 + vd->vd_meta.lvm_name.get()
                                 + ": cannot find action -- " + (*act_iter));
#endif
            }
        }

        vd->set_rewrite_src_name();
    }

    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
        for (const auto& vd : this->elf_value_def_order) {
            this->elf_value_def_frag_map[vd->vd_meta.lvm_name
                                             .to_string_fragment()]
                = vd.get();
        }
    }

    for (const auto& td_pair : this->lf_tag_defs) {
        const auto& td = td_pair.second;

        if (td->ftd_pattern.pp_value == nullptr
            || td->ftd_pattern.pp_value->get_pattern().empty())
        {
            errors.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t("invalid tag definition ")
                        .append_quoted(lnav::roles::symbol(
                            fmt::format(FMT_STRING("/{}/tags/{}"),
                                        this->elf_name,
                                        td_pair.first))))
                    .with_reason(
                        "tag definitions must have a non-empty pattern")
                    .with_snippets(this->get_snippets()));
        }
    }

    for (const auto& opid_desc_pair : *this->lf_opid_description_def) {
        for (const auto& opid_desc : *opid_desc_pair.second.od_descriptors) {
            auto iter = this->elf_value_defs.find(opid_desc.od_field.pp_value);
            if (iter == this->elf_value_defs.end()) {
                errors.emplace_back(
                    lnav::console::user_message::error(
                        attr_line_t("invalid opid description field ")
                            .append_quoted(lnav::roles::symbol(
                                opid_desc.od_field.pp_path.to_string())))
                        .with_reason(
                            attr_line_t("unknown value name ")
                                .append_quoted(opid_desc.od_field.pp_value))
                        .with_snippets(this->get_snippets()));
            } else {
                this->lf_desc_fields.insert(iter->first);
                iter->second->vd_is_desc_field = true;
            }
        }
    }

    for (const auto& subid_desc_pair : *this->lf_subid_description_def) {
        for (const auto& subid_desc : *subid_desc_pair.second.od_descriptors) {
            auto iter = this->elf_value_defs.find(subid_desc.od_field.pp_value);
            if (iter == this->elf_value_defs.end()) {
                errors.emplace_back(
                    lnav::console::user_message::error(
                        attr_line_t("invalid subid description field ")
                            .append_quoted(lnav::roles::symbol(
                                subid_desc.od_field.pp_path.to_string())))
                        .with_reason(
                            attr_line_t("unknown value name ")
                                .append_quoted(subid_desc.od_field.pp_value))
                        .with_snippets(this->get_snippets()));
            } else {
                this->lf_desc_fields.insert(iter->first);
                iter->second->vd_is_desc_field = true;
            }
        }
    }

    if (this->elf_type == elf_type_t::ELF_TYPE_TEXT
        && this->elf_samples.empty())
    {
        errors.emplace_back(
            lnav::console::user_message::error(
                attr_line_t()
                    .append_quoted(
                        lnav::roles::symbol(this->elf_name.to_string()))
                    .append(" is not a valid log format"))
                .with_reason("log message samples must be included in a format "
                             "definition")
                .with_snippets(this->get_snippets()));
    }

    for (const auto& pat : this->elf_pattern_order) {
        if (pat->p_module_format || this->elf_type != elf_type_t::ELF_TYPE_TEXT)
        {
            continue;
        }
        if (pat->p_pcre.pp_value->name_index(this->lf_timestamp_field.get())
            < 0)
        {
            attr_line_t notes;
            bool first_note = true;

            if (pat->p_pcre.pp_value->get_capture_count() > 0) {
                notes.append("the following captures are available:\n  ");
            }
            for (auto named_cap : pat->p_pcre.pp_value->get_named_captures()) {
                if (!first_note) {
                    notes.append(", ");
                }
                notes.append(
                    lnav::roles::symbol(named_cap.get_name().to_string()));
                first_note = false;
            }
            errors.emplace_back(
                lnav::console::user_message::error(
                    attr_line_t("invalid value for property ")
                        .append_quoted(lnav::roles::symbol(
                            fmt::format(FMT_STRING("/{}/timestamp-field"),
                                        this->elf_name))))
                    .with_reason(
                        attr_line_t()
                            .append_quoted(this->lf_timestamp_field)
                            .append(" was not found in the pattern at ")
                            .append(lnav::roles::symbol(pat->p_config_path)))
                    .with_note(notes)
                    .with_snippets(this->get_snippets()));
        }
    }

    for (size_t sample_index = 0; sample_index < this->elf_samples.size();
         sample_index += 1)
    {
        auto& elf_sample = this->elf_samples[sample_index];
        auto sample_lines
            = string_fragment(elf_sample.s_line.pp_value).split_lines();

        if (this->test_line(elf_sample, errors).is<scan_match>()) {
            for (const auto& pat_name : elf_sample.s_matched_regexes) {
                this->elf_patterns[pat_name]->p_matched_samples.emplace(
                    sample_index);
            }
        }
    }

    if (!this->elf_samples.empty()) {
        for (const auto& elf_sample : this->elf_samples) {
            if (elf_sample.s_matched_regexes.size() <= 1) {
                continue;
            }

            errors.emplace_back(
                lnav::console::user_message::warning(
                    attr_line_t("invalid log format: ")
                        .append_quoted(
                            lnav::roles::symbol(this->elf_name.to_string())))
                    .with_reason(
                        attr_line_t(
                            "sample is matched by more than one regex: ")
                            .join(elf_sample.s_matched_regexes,
                                  VC_ROLE.value(role_t::VCR_SYMBOL),
                                  ", "))
                    .with_snippet(lnav::console::snippet::from(
                        elf_sample.s_line.pp_location,
                        attr_line_t().append(lnav::roles::quoted_code(
                            elf_sample.s_line.pp_value))))
                    .with_help("log format regexes must match a single type "
                               "of log message"));
        }

        for (const auto& pat : this->elf_pattern_order) {
            if (pat->p_module_format) {
                continue;
            }

            if (pat->p_matched_samples.empty()) {
                errors.emplace_back(
                    lnav::console::user_message::warning(
                        attr_line_t("invalid pattern: ")
                            .append_quoted(
                                lnav::roles::symbol(pat->p_config_path)))
                        .with_reason("pattern does not match any samples")
                        .with_snippet(lnav::console::snippet::from(
                            pat->p_pcre.pp_location, ""))
                        .with_help(
                            "every pattern should have at least one sample "
                            "that it matches"));
            }
        }
    }

    size_t value_def_index = 0;
    for (auto& elf_value_def : this->elf_value_def_order) {
        elf_value_def->vd_meta.lvm_values_index
            = std::make_optional(value_def_index++);

        if (elf_value_def->vd_meta.lvm_foreign_key
            || elf_value_def->vd_meta.lvm_identifier)
        {
            continue;
        }

        switch (elf_value_def->vd_meta.lvm_kind) {
            case value_kind_t::VALUE_INTEGER:
            case value_kind_t::VALUE_FLOAT:
                this->elf_numeric_value_defs.push_back(elf_value_def);
                break;
            default:
                break;
        }
    }

    int format_index = 0;
    for (auto iter = this->jlf_line_format.begin();
         iter != this->jlf_line_format.end();
         ++iter, format_index++)
    {
        static const intern_string_t ts
            = intern_string::lookup("__timestamp__");
        static const intern_string_t level_field
            = intern_string::lookup("__level__");
        auto& jfe = *iter;

        if (startswith(jfe.jfe_value.pp_value.get(), "/")) {
            jfe.jfe_value.pp_value
                = intern_string::lookup(jfe.jfe_value.pp_value.get() + 1);
        }
        if (!jfe.jfe_ts_format.empty()) {
            if (!jfe.jfe_value.pp_value.empty() && jfe.jfe_value.pp_value != ts)
            {
                log_warning(
                    "%s:line-format[%d]:ignoring field '%s' since "
                    "timestamp-format was used",
                    this->elf_name.get(),
                    format_index,
                    jfe.jfe_value.pp_value.get());
            }
            jfe.jfe_value.pp_value = ts;
        }

        switch (jfe.jfe_type) {
            case json_log_field::VARIABLE: {
                auto vd_iter
                    = this->elf_value_defs.find(jfe.jfe_value.pp_value);
                if (jfe.jfe_value.pp_value == ts) {
                    this->elf_value_defs[this->lf_timestamp_field]
                        ->vd_meta.lvm_hidden
                        = true;
                } else if (jfe.jfe_value.pp_value == level_field) {
                    this->elf_value_defs[this->elf_level_field]
                        ->vd_meta.lvm_hidden
                        = true;
                } else if (vd_iter == this->elf_value_defs.end()) {
                    errors.emplace_back(
                        lnav::console::user_message::error(
                            attr_line_t("invalid line format element ")
                                .append_quoted(lnav::roles::symbol(fmt::format(
                                    FMT_STRING("/{}/line-format/{}/field"),
                                    this->elf_name,
                                    format_index))))
                            .with_reason(
                                attr_line_t()
                                    .append_quoted(jfe.jfe_value.pp_value)
                                    .append(" is not a defined value"))
                            .with_snippet(jfe.jfe_value.to_snippet()));
                } else {
                    vd_iter->second->vd_line_format_index = format_index;
                    switch (vd_iter->second->vd_meta.lvm_kind) {
                        case value_kind_t::VALUE_INTEGER:
                        case value_kind_t::VALUE_FLOAT:
                            if (jfe.jfe_align
                                == json_format_element::align_t::NONE)
                            {
                                jfe.jfe_align
                                    = json_format_element::align_t::RIGHT;
                            }
                            break;
                        default:
                            break;
                    }
                }
                break;
            }
            case json_log_field::CONSTANT:
                this->jlf_line_format_init_count
                    += std::count(jfe.jfe_default_value.begin(),
                                  jfe.jfe_default_value.end(),
                                  '\n');
                break;
            default:
                break;
        }
    }

    for (auto& hd_pair : this->elf_highlighter_patterns) {
        auto& hd = hd_pair.second;
        text_attrs attrs;

        if (!hd.hd_color.pp_value.empty()) {
            attrs.ta_fg_color = vc.match_color(
                styling::color_unit::from_str(hd.hd_color.pp_value)
                    .unwrapOrElse([&](const auto& msg) {
                        errors.emplace_back(
                            lnav::console::user_message::error(
                                attr_line_t()
                                    .append_quoted(hd.hd_color.pp_value)
                                    .append(" is not a valid color value for "
                                            "property ")
                                    .append_quoted(lnav::roles::symbol(
                                        hd.hd_color.pp_path.to_string())))
                                .with_reason(msg)
                                .with_snippet(hd.hd_color.to_snippet()));
                        return styling::color_unit::EMPTY;
                    }));
        }

        if (!hd.hd_background_color.pp_value.empty()) {
            attrs.ta_bg_color = vc.match_color(
                styling::color_unit::from_str(hd.hd_background_color.pp_value)
                    .unwrapOrElse([&](const auto& msg) {
                        errors.emplace_back(
                            lnav::console::user_message::error(
                                attr_line_t()
                                    .append_quoted(
                                        hd.hd_background_color.pp_value)
                                    .append(" is not a valid color value for "
                                            "property ")
                                    .append_quoted(lnav::roles::symbol(
                                        hd.hd_background_color.pp_path
                                            .to_string())))
                                .with_reason(msg)
                                .with_snippet(
                                    hd.hd_background_color.to_snippet()));
                        return styling::color_unit::EMPTY;
                    }));
        }

        if (hd.hd_underline) {
            attrs |= text_attrs::style::underline;
        }
        if (hd.hd_blink) {
            attrs |= text_attrs::style::blink;
        }

        if (hd.hd_pattern.pp_value != nullptr) {
            this->lf_highlighters.emplace_back(hd.hd_pattern.pp_value);
            this->lf_highlighters.back()
                .with_name(hd_pair.first.to_string())
                .with_format_name(this->elf_name)
                .with_attrs(attrs);
        }
    }

    this->lf_value_stats.resize(this->elf_value_defs.size());
}

void
external_log_format::register_vtabs(
    log_vtab_manager* vtab_manager,
    std::vector<lnav::console::user_message>& errors)
{
    for (auto& elf_search_table : this->elf_search_tables) {
        if (elf_search_table.second.std_pattern.pp_value == nullptr) {
            continue;
        }

        auto lst = std::make_shared<log_search_table>(
            elf_search_table.second.std_pattern.pp_value,
            elf_search_table.first);
        lst->lst_format = this;
        lst->lst_log_path_glob = elf_search_table.second.std_glob;
        if (elf_search_table.second.std_level != LEVEL_UNKNOWN) {
            lst->lst_log_level = elf_search_table.second.std_level;
        }
        auto errmsg = vtab_manager->register_vtab(lst);
        if (!errmsg.empty()) {
#if 0
            errors.push_back("error:" + this->elf_name.to_string() + ":"
                             + search_iter->first.to_string()
                             + ":unable to register table -- " + errmsg);
#endif
        }
    }
}

bool
external_log_format::match_samples(const std::vector<sample_t>& samples) const
{
    for (const auto& sample_iter : samples) {
        for (const auto& pat_iter : this->elf_pattern_order) {
            auto& pat = *pat_iter;

            if (!pat.p_pcre.pp_value) {
                continue;
            }

            if (pat.p_pcre.pp_value
                    ->find_in(sample_iter.s_line.pp_value, PCRE2_NO_UTF_CHECK)
                    .ignore_error())
            {
                return true;
            }
        }
    }

    return false;
}

class external_log_table : public log_format_vtab_impl {
public:
    explicit external_log_table(const external_log_format& elf)
        : log_format_vtab_impl(elf), elt_format(elf)
    {
    }

    void get_columns(std::vector<vtab_column>& cols) const override
    {
        const auto& elf = this->elt_format;

        cols.resize(elf.elf_column_count);
        for (const auto& vd : elf.elf_value_def_order) {
            auto type_pair = log_vtab_impl::logline_value_to_sqlite_type(
                vd->vd_meta.lvm_kind);

            if (!vd->vd_meta.lvm_column.is<logline_value_meta::table_column>())
            {
                continue;
            }

            auto col
                = vd->vd_meta.lvm_column.get<logline_value_meta::table_column>()
                      .value;
            require(0 <= col && col < elf.elf_column_count);

            cols[col].vc_name = vd->vd_meta.lvm_name.get();
            cols[col].vc_type = type_pair.first;
            cols[col].vc_subtype = type_pair.second;
            cols[col].vc_collator = vd->vd_collate;
            cols[col].vc_comment = vd->vd_description;
        }
    }

    void get_foreign_keys(
        std::unordered_set<std::string>& keys_inout) const override
    {
        log_vtab_impl::get_foreign_keys(keys_inout);

        for (const auto& elf_value_def : this->elt_format.elf_value_defs) {
            if (elf_value_def.second->vd_meta.lvm_foreign_key
                || elf_value_def.second->vd_meta.lvm_identifier)
            {
                keys_inout.emplace(elf_value_def.first.to_string());
            }
        }
    }

    bool next(log_cursor& lc, logfile_sub_source& lss) override
    {
        if (lc.is_eof()) {
            return true;
        }

        content_line_t cl(lss.at(lc.lc_curr_line));
        auto* lf = lss.find_file_ptr(cl);
        auto lf_iter = lf->begin() + cl;
        uint8_t mod_id = lf_iter->get_module_id();

        if (lf_iter->is_continued()) {
            return false;
        }

        this->elt_module_format.mf_mod_format = nullptr;
        if (lf->get_format_name() == this->lfvi_format.get_name()) {
            return true;
        }
        if (mod_id && mod_id == this->lfvi_format.lf_mod_index) {
            auto format = lf->get_format();

            return lf->read_line(lf_iter)
                .map([this, format, cl, lf](auto line) {
                    string_attrs_t sa;
                    logline_value_vector values;
                    line_range mod_name_range;
                    intern_string_t mod_name;

                    values.lvv_sbr = line.clone();
                    format->annotate(lf, cl, sa, values, false);
                    this->elt_container_body
                        = find_string_attr_range(sa, &SA_BODY);
                    if (!this->elt_container_body.is_valid()) {
                        return false;
                    }
                    this->elt_container_body.ltrim(line.get_data());
                    mod_name_range = find_string_attr_range(sa, &L_MODULE);
                    if (!mod_name_range.is_valid()) {
                        return false;
                    }
                    mod_name = intern_string::lookup(
                        &line.get_data()[mod_name_range.lr_start],
                        mod_name_range.length());
                    this->elt_module_format
                        = external_log_format::MODULE_FORMATS[mod_name];
                    if (!this->elt_module_format.mf_mod_format) {
                        return false;
                    }
                    return this->elt_module_format.mf_mod_format->get_name()
                        == this->lfvi_format.get_name();
                })
                .unwrapOr(false);
        }

        return false;
    }

    void extract(logfile* lf,
                 uint64_t line_number,
                 string_attrs_t& sa,
                 logline_value_vector& values) override
    {
        auto& line = values.lvv_sbr;
        auto format = lf->get_format();

        if (this->elt_module_format.mf_mod_format != nullptr) {
            shared_buffer_ref body_ref;

            body_ref.subset(line,
                            this->elt_container_body.lr_start,
                            this->elt_container_body.length());
            sa.clear();
            auto narrow_res
                = values.lvv_sbr.narrow(this->elt_container_body.lr_start,
                                        this->elt_container_body.length());
            values.lvv_values.clear();
            this->elt_module_format.mf_mod_format->annotate(
                lf, line_number, sa, values, false);
            values.lvv_sbr.widen(narrow_res);
        } else {
            sa.clear();
            format->annotate(lf, line_number, sa, values, false);
        }
    }

    const external_log_format& elt_format;
    module_format elt_module_format;
    struct line_range elt_container_body;
};

std::shared_ptr<log_vtab_impl>
external_log_format::get_vtab_impl() const
{
    return std::make_shared<external_log_table>(*this);
}

std::shared_ptr<log_format>
external_log_format::specialized(int fmt_lock)
{
    auto retval = std::make_shared<external_log_format>(*this);

    retval->lf_specialized = true;
    this->lf_pattern_locks.clear();
    if (fmt_lock != -1) {
        retval->lf_pattern_locks.emplace_back(0, fmt_lock);
    }

    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
        this->jlf_parse_context
            = std::make_shared<yajlpp_parse_context>(this->elf_name);
        this->jlf_yajl_handle.reset(
            yajl_alloc(&this->jlf_parse_context->ypc_callbacks,
                       nullptr,
                       this->jlf_parse_context.get()),
            yajl_handle_deleter());
        yajl_config(this->jlf_yajl_handle.get(), yajl_dont_validate_strings, 1);
        this->jlf_cached_line.reserve(16 * 1024);
    }

    this->lf_value_stats.clear();
    this->lf_value_stats.resize(this->elf_value_defs.size());
    this->elf_specialized_value_defs_state = *this->elf_value_defs_state;

    return retval;
}

log_format::match_name_result
external_log_format::match_name(const std::string& filename)
{
    if (this->elf_filename_pcre.pp_value == nullptr) {
        return name_matched{};
    }

    if (this->elf_filename_pcre.pp_value->find_in(filename)
            .ignore_error()
            .has_value())
    {
        return name_matched{};
    }

    return name_mismatched{
        this->elf_filename_pcre.pp_value->match_partial(filename),
        this->elf_filename_pcre.pp_value->get_pattern(),
    };
}

auto
external_log_format::value_line_count(const value_def* vd,
                                      bool top_level,
                                      std::optional<double> val,
                                      const unsigned char* str,
                                      ssize_t len,
                                      yajl_string_props_t* props)
    -> value_line_count_result
{
    value_line_count_result retval;

    if (vd == nullptr) {
        if (this->jlf_hide_extra || !top_level) {
            retval.vlcr_count = 0;
        }

        return retval;
    }

    if (str != nullptr && props != nullptr && !val) {
        auto frag = string_fragment::from_bytes(str, len);
        while (frag.endswith("\n")) {
            frag.pop_back();
            props->line_feeds -= 1;
        }
        retval.vlcr_has_ansi |= props->has_ansi;
        retval.vlcr_count += props->line_feeds;
    }

    if (vd->vd_meta.lvm_values_index) {
        auto& lvs = this->lf_value_stats[vd->vd_meta.lvm_values_index.value()];
        if (len > lvs.lvs_width) {
            lvs.lvs_width = len;
        }
        if (val) {
            lvs.add_value(val.value());
        }
    }

    if (vd->vd_line_format_index) {
        retval.vlcr_line_format_count += 1;
        retval.vlcr_count -= 1;
        retval.vlcr_line_format_index = vd->vd_line_format_index;
    } else if (vd->vd_meta.is_hidden()) {
        retval.vlcr_count = 0;
        return retval;
    }

    return retval;
}

log_level_t
external_log_format::convert_level(string_fragment sf,
                                   scan_batch_context* sbc) const
{
    auto retval = LEVEL_INFO;

    if (sf.is_valid()) {
        if (sbc != nullptr) {
            auto ssm_res = sbc->sbc_level_cache.lookup(sf);
            if (ssm_res.has_value()) {
                return static_cast<log_level_t>(ssm_res.value());
            }
        }

        if (this->elf_level_patterns.empty()) {
            retval = string2level(sf.data(), sf.length());
        } else {
            for (const auto& elf_level_pattern : this->elf_level_patterns) {
                if (elf_level_pattern.second.lp_pcre.pp_value
                        ->find_in(sf, PCRE2_NO_UTF_CHECK)
                        .ignore_error()
                        .has_value())
                {
                    retval = elf_level_pattern.first;
                    break;
                }
            }
        }

        if (sbc != nullptr
            && sf.length() <= lnav::small_string_map::MAX_KEY_SIZE)
        {
            sbc->sbc_level_cache.insert(sf, retval);
        }
    }

    return retval;
}

logline_value_meta
external_log_format::get_value_meta(intern_string_t field_name,
                                    value_kind_t kind)
{
    const auto iter = this->elf_value_defs.find(field_name);
    if (iter == this->elf_value_defs.end()) {
        auto retval = logline_value_meta(
            field_name, kind, logline_value_meta::external_column{}, this);

        retval.lvm_hidden = this->jlf_hide_extra;
        return retval;
    }

    auto lvm = iter->second->vd_meta;

    lvm.lvm_kind = kind;
    return lvm;
}

logline_value_meta
external_log_format::get_value_meta(yajlpp_parse_context* ypc,
                                    const value_def* vd,
                                    value_kind_t kind)
{
    if (vd == nullptr) {
        auto retval = logline_value_meta(
            ypc->get_path(), kind, logline_value_meta::external_column{}, this);

        retval.lvm_hidden = this->jlf_hide_extra;
        return retval;
    }

    auto lvm = vd->vd_meta;

    lvm.lvm_kind = kind;
    return lvm;
}

void
external_log_format::json_append(const json_format_element& jfe,
                                 const value_def* vd,
                                 const string_fragment& sf)
{
    if (jfe.jfe_align == json_format_element::align_t::RIGHT) {
        auto sf_width = sf.column_width();
        if (sf_width < jfe.jfe_min_width) {
            this->json_append_to_cache(jfe.jfe_min_width - sf_width);
        } else if (jfe.jfe_auto_width && vd != nullptr
                   && sf_width
                       < this->lf_value_stats[vd->vd_meta.lvm_values_index
                                                  .value()]
                             .lvs_width)
        {
            this->json_append_to_cache(
                this->lf_value_stats[vd->vd_meta.lvm_values_index.value()]
                    .lvs_width
                - sf_width);
        }
    }
    this->json_append_to_cache(sf.data(), sf.length());
    if ((jfe.jfe_align == json_format_element::align_t::LEFT
         || jfe.jfe_align == json_format_element::align_t::NONE)
        && (jfe.jfe_min_width > 0 || jfe.jfe_auto_width))
    {
        auto sf_width = sf.column_width();
        if (sf_width < jfe.jfe_min_width) {
            this->json_append_to_cache(jfe.jfe_min_width - sf_width);
        } else if (jfe.jfe_auto_width && vd != nullptr
                   && sf_width
                       < this->lf_value_stats[vd->vd_meta.lvm_values_index
                                                  .value()]
                             .lvs_width)
        {
            this->json_append_to_cache(
                this->lf_value_stats[vd->vd_meta.lvm_values_index.value()]
                    .lvs_width
                - sf_width);
        }
    }
}

intern_string_t
external_log_format::get_pattern_name(uint64_t line_number) const
{
    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
        static auto structured = intern_string::lookup("structured");

        return structured;
    }
    int pat_index = this->pattern_index_for_line(line_number);
    return this->elf_pattern_order[pat_index]->p_name;
}

int
log_format::pattern_index_for_line(uint64_t line_number) const
{
    if (this->lf_pattern_locks.empty()) {
        return -1;
    }

    auto iter = lower_bound(this->lf_pattern_locks.cbegin(),
                            this->lf_pattern_locks.cend(),
                            line_number,
                            [](const pattern_for_lines& pfl, uint32_t line) {
                                return pfl.pfl_line < line;
                            });

    if (iter == this->lf_pattern_locks.end() || iter->pfl_line != line_number) {
        --iter;
    }

    return iter->pfl_pat_index;
}

std::string
log_format::get_pattern_path(uint64_t line_number) const
{
    int pat_index = this->pattern_index_for_line(line_number);
    return fmt::format(FMT_STRING("builtin ({})"), pat_index);
}

intern_string_t
log_format::get_pattern_name(uint64_t line_number) const
{
    char pat_str[128];

    int pat_index = this->pattern_index_for_line(line_number);
    auto to_n_res = fmt::format_to_n(
        pat_str, sizeof(pat_str) - 1, FMT_STRING("builtin ({})"), pat_index);
    pat_str[to_n_res.size] = '\0';
    return intern_string::lookup(pat_str);
}

std::shared_ptr<log_format>
log_format::find_root_format(const char* name)
{
    auto& fmts = get_root_formats();
    for (auto& lf : fmts) {
        if (lf->get_name() == name) {
            return lf;
        }
    }
    return nullptr;
}

exttm
log_format::tm_for_display(logfile::iterator ll, string_fragment sf)
{
    auto adjusted_time = ll->get_timeval();
    exttm retval;

    retval.et_nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(
                         std::chrono::microseconds{adjusted_time.tv_usec})
                         .count();
    if (this->lf_timestamp_flags & ETF_NANOS_SET) {
        timeval actual_tv;
        exttm tm;
        if (this->lf_date_time.scan(sf.data(),
                                    sf.length(),
                                    this->get_timestamp_formats(),
                                    &tm,
                                    actual_tv,
                                    false))
        {
            adjusted_time.tv_usec = actual_tv.tv_usec;
            retval.et_nsec = tm.et_nsec;
        }
    }
    gmtime_r(&adjusted_time.tv_sec, &retval.et_tm);
    retval.et_flags = this->lf_timestamp_flags;
    if (this->lf_timestamp_flags & ETF_ZONE_SET
        && this->lf_date_time.dts_zoned_to_local)
    {
        retval.et_flags &= ~ETF_Z_IS_UTC;
    }
    retval.et_gmtoff = this->lf_date_time.dts_local_offset_cache;

    return retval;
}

log_format::pattern_for_lines::pattern_for_lines(uint32_t pfl_line,
                                                 uint32_t pfl_pat_index)
    : pfl_line(pfl_line), pfl_pat_index(pfl_pat_index)
{
}

void
logline_value_stats::merge(const logline_value_stats& other)
{
    if (other.lvs_count == 0) {
        return;
    }

    require(other.lvs_min_value <= other.lvs_max_value);

    if (other.lvs_width > this->lvs_width) {
        this->lvs_width = other.lvs_width;
    }
    if (other.lvs_min_value < this->lvs_min_value) {
        this->lvs_min_value = other.lvs_min_value;
    }
    if (other.lvs_max_value > this->lvs_max_value) {
        this->lvs_max_value = other.lvs_max_value;
    }
    this->lvs_count += other.lvs_count;
    this->lvs_total += other.lvs_total;

    ensure(this->lvs_count >= 0);
    ensure(this->lvs_min_value <= this->lvs_max_value);
}

void
logline_value_stats::add_value(double value)
{
    if (value < this->lvs_min_value) {
        this->lvs_min_value = value;
    }
    if (value > this->lvs_max_value) {
        this->lvs_max_value = value;
    }
    this->lvs_count += 1;
    this->lvs_total += value;
}

std::vector<logline_value_meta>
external_log_format::get_value_metadata() const
{
    std::vector<logline_value_meta> retval;

    for (const auto& vd : this->elf_value_def_order) {
        retval.emplace_back(vd->vd_meta);
    }

    return retval;
}

const logline_value_stats*
external_log_format::stats_for_value(const intern_string_t& name) const
{
    const auto iter = this->elf_value_defs.find(name);
    if (iter != this->elf_value_defs.end()
        && iter->second->vd_meta.lvm_values_index)
    {
        return &this->lf_value_stats[iter->second->vd_meta.lvm_values_index
                                         .value()];
    }

    return nullptr;
}

std::string
external_log_format::get_pattern_regex(uint64_t line_number) const
{
    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
        return "";
    }
    int pat_index = this->pattern_index_for_line(line_number);
    return this->elf_pattern_order[pat_index]->p_pcre.pp_value->get_pattern();
}

bool
external_log_format::hide_field(const intern_string_t field_name, bool val)
{
    const auto vd_iter = this->elf_value_defs.find(field_name);
    if (vd_iter == this->elf_value_defs.end()) {
        return false;
    }

    vd_iter->second->vd_meta.lvm_user_hidden = val;
    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
        bool found = false;

        for (const auto& jfe : this->jlf_line_format) {
            if (jfe.jfe_value.pp_value == field_name) {
                found = true;
            }
        }
        if (!found) {
            log_info("format field %s.%s changed, rebuilding",
                     this->elf_name.get(),
                     field_name.get());
            this->elf_value_defs_state->vds_generation += 1;
        }
    }
    return true;
}

bool
external_log_format::format_changed()
{
    if (this->elf_specialized_value_defs_state.vds_generation
        != this->elf_value_defs_state->vds_generation)
    {
        this->elf_specialized_value_defs_state = *this->elf_value_defs_state;
        this->jlf_cached_offset = -1;
        return true;
    }

    return false;
}

bool
format_tag_def::path_restriction::matches(const char* fn) const
{
    return fnmatch(this->p_glob.c_str(), fn, 0) == 0;
}

bool
format_partition_def::path_restriction::matches(const char* fn) const
{
    return fnmatch(this->p_glob.c_str(), fn, 0) == 0;
}

/* XXX */
#include "log_format_impls.cc"

tstack / lnav / 17589970077-2502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous