17589970077-2502

Committed 09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%

Build # 17589970077-2502

Build Type

push

github

Committed by

tstack

Commit Message

[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

Run Details

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.65

/src/log_format.hh

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file log_format.hh
 */

#ifndef log_format_hh
#define log_format_hh

#include <stdint.h>
#include <sys/time.h>
#include <time.h>
#define __STDC_FORMAT_MACROS
#include <limits>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include <inttypes.h>
#include <sys/types.h>

#include "base/date_time_scanner.hh"
#include "base/intern_string.hh"
#include "base/lnav_log.hh"
#include "highlighter.hh"
#include "line_buffer.hh"
#include "log_format_fwd.hh"
#include "logfile.hh"
#include "pcrepp/pcre2pp.hh"
#include "robin_hood/robin_hood.h"
#include "shared_buffer.hh"

struct sqlite3;
class logfile;
class log_vtab_manager;
struct exec_context;

enum class scale_op_t {
    SO_IDENTITY,
    SO_MULTIPLY,
    SO_DIVIDE
};

struct scaling_factor {
    template<typename T>
    void scale(T& val) const
    {
        switch (this->sf_op) {
            case scale_op_t::SO_IDENTITY:
                break;
            case scale_op_t::SO_DIVIDE:
                val = val / (T) this->sf_value;
                break;
            case scale_op_t::SO_MULTIPLY:
                val = val * (T) this->sf_value;
                break;
        }
    }

    scale_op_t sf_op{scale_op_t::SO_IDENTITY};
    double sf_value{1};
};

enum class value_kind_t : int {
    VALUE_UNKNOWN = -1,
    VALUE_NULL,
    VALUE_TEXT,
    VALUE_INTEGER,
    VALUE_FLOAT,
    VALUE_BOOLEAN,
    VALUE_JSON,
    VALUE_STRUCT,
    VALUE_QUOTED,
    VALUE_W3C_QUOTED,
    VALUE_TIMESTAMP,
    VALUE_XML,

    VALUE__MAX
};

enum class chart_type_t {
    none,
    hist,
    spectro,
};

struct logline_value_meta {
    struct internal_column {
        bool operator==(const internal_column&) const { return true; }
    };
    struct external_column {
        bool operator==(const external_column&) const { return true; }
    };
    struct table_column {
        size_t value;

        bool operator==(const table_column& rhs) const
        {
            return this->value == rhs.value;
        }
    };

    using column_t
        = mapbox::util::variant<internal_column, external_column, table_column>;

    logline_value_meta(intern_string_t name,
                       value_kind_t kind,
                       column_t col = external_column{},
                       const std::optional<log_format*>& format = std::nullopt)
        : lvm_name(name), lvm_kind(kind), lvm_column(col), lvm_format(format)
    {
    }

    bool is_hidden() const
    {
        if (this->lvm_user_hidden) {
            return this->lvm_user_hidden.value();
        }
        return this->lvm_hidden;
    }

    bool is_numeric() const
    {
        if (this->lvm_identifier || this->lvm_foreign_key) {
            return false;
        }
        switch (this->lvm_kind) {
            case value_kind_t::VALUE_FLOAT:
            case value_kind_t::VALUE_INTEGER:
                return true;
            default:
                return false;
        }
    }

    logline_value_meta& with_struct_name(intern_string_t name)
    {
        this->lvm_struct_name = name;
        return *this;
    }

    chart_type_t to_chart_type() const;

    intern_string_t lvm_name;
    value_kind_t lvm_kind;
    column_t lvm_column{external_column{}};
    std::optional<size_t> lvm_values_index;
    bool lvm_identifier{false};
    bool lvm_foreign_key{false};
    bool lvm_hidden{false};
    std::optional<bool> lvm_user_hidden;
    bool lvm_from_module{false};
    intern_string_t lvm_struct_name;
    std::optional<log_format*> lvm_format;
};

class logline_value {
public:
    logline_value(logline_value_meta lvm) : lv_meta(std::move(lvm))
    {
        this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL;
    }

    logline_value(logline_value_meta lvm, bool b)
        : lv_meta(std::move(lvm)), lv_value((int64_t) (b ? 1 : 0))
    {
        this->lv_meta.lvm_kind = value_kind_t::VALUE_BOOLEAN;
    }

    logline_value(logline_value_meta lvm, int64_t i)
        : lv_meta(std::move(lvm)), lv_value(i)
    {
        this->lv_meta.lvm_kind = value_kind_t::VALUE_INTEGER;
    }

    logline_value(logline_value_meta lvm, double i)
        : lv_meta(std::move(lvm)), lv_value(i)
    {
        this->lv_meta.lvm_kind = value_kind_t::VALUE_FLOAT;
    }

    logline_value(logline_value_meta lvm, string_fragment frag)
        : lv_meta(std::move(lvm)), lv_frag(frag)
    {
    }

    logline_value(logline_value_meta lvm, const intern_string_t val)
        : lv_meta(std::move(lvm)), lv_intern_string(val)
    {
    }

    logline_value(logline_value_meta lvm, std::string val)
        : lv_meta(std::move(lvm)), lv_str(std::move(val))
    {
    }

    logline_value(logline_value_meta lvm,
                  shared_buffer_ref& sbr,
                  struct line_range origin);

    void apply_scaling(const scaling_factor* sf)
    {
        if (sf != nullptr) {
            switch (this->lv_meta.lvm_kind) {
                case value_kind_t::VALUE_INTEGER:
                    sf->scale(this->lv_value.i);
                    break;
                case value_kind_t::VALUE_FLOAT:
                    sf->scale(this->lv_value.d);
                    break;
                default:
                    break;
            }
        }
    }

    std::string to_string() const;

    string_fragment to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const;

    const char* text_value() const
    {
        if (this->lv_str) {
            return this->lv_str->c_str();
        }
        if (this->lv_frag.empty()) {
            if (this->lv_intern_string.empty()) {
                return "";
            }
            return this->lv_intern_string.get();
        }
        return this->lv_frag.data();
    }

    size_t text_length() const
    {
        if (this->lv_str) {
            return this->lv_str->size();
        }
        if (this->lv_frag.empty()) {
            return this->lv_intern_string.size();
        }
        return this->lv_frag.length();
    }

    struct line_range origin_in_full_msg(const char* msg, ssize_t len) const;

    logline_value_meta lv_meta;
    union value_u {
        int64_t i;
        double d;

        value_u() : i(0) {}
        value_u(int64_t i) : i(i) {}
        value_u(double d) : d(d) {}
    } lv_value;
    std::optional<std::string> lv_str;
    string_fragment lv_frag;
    int lv_sub_offset{0};
    intern_string_t lv_intern_string;
    struct line_range lv_origin;
};

struct logline_value_vector {
    enum class opid_provenance {
        none,
        file,
        user,
    };

    void clear()
    {
        this->lvv_values.clear();
        this->lvv_sbr.disown();
        this->lvv_opid_value = std::nullopt;
        this->lvv_opid_provenance = opid_provenance::none;
    }

    logline_value_vector() = default;

    logline_value_vector(const logline_value_vector& other)
        : lvv_sbr(other.lvv_sbr.clone()), lvv_values(other.lvv_values),
          lvv_opid_value(other.lvv_opid_value),
          lvv_opid_provenance(other.lvv_opid_provenance)
    {
    }

    logline_value_vector& operator=(const logline_value_vector& other)
    {
        this->lvv_sbr = other.lvv_sbr.clone();
        this->lvv_values = other.lvv_values;
        this->lvv_opid_value = other.lvv_opid_value;
        this->lvv_opid_provenance = other.lvv_opid_provenance;

        return *this;
    }

    shared_buffer_ref lvv_sbr;
    std::vector<logline_value> lvv_values;
    std::optional<std::string> lvv_opid_value;
    opid_provenance lvv_opid_provenance{opid_provenance::none};
};

struct logline_value_stats {
    logline_value_stats() { this->clear(); }

    void clear()
    {
        this->lvs_width = 0;
        this->lvs_count = 0;
        this->lvs_total = 0;
        this->lvs_min_value = std::numeric_limits<double>::max();
        this->lvs_max_value = -std::numeric_limits<double>::max();
    }

    void merge(const logline_value_stats& other);

    void add_value(double value);

    int64_t lvs_width;
    int64_t lvs_count;
    double lvs_total;
    double lvs_min_value;
    double lvs_max_value;
};

struct logline_value_name_cmp {
    explicit logline_value_name_cmp(const intern_string_t* name)
        : lvc_name(name)
    {
    }

    bool operator()(const logline_value& lv) const
    {
        return (*this->lvc_name) == lv.lv_meta.lvm_name;
    }

    const intern_string_t* lvc_name;
};

struct logline_value_col_eq {
    explicit logline_value_col_eq(const logline_value_meta::table_column& col)
        : lvc_column(col)
    {
    }

    bool operator()(const logline_value& lv) const
    {
        if (lv.lv_meta.lvm_column.is<logline_value_meta::table_column>()) {
            return this->lvc_column
                == lv.lv_meta.lvm_column
                       .get<logline_value_meta::table_column>();
        }
        return false;
    }

    const logline_value_meta::table_column& lvc_column;
};

class log_vtab_impl;

/**
 * Base class for implementations of log format parsers.
 */
class log_format {
public:
    /**
     * @return The collection of builtin log formats.
     */
    static std::vector<std::shared_ptr<log_format>>& get_root_formats();

    static std::shared_ptr<log_format> find_root_format(const char* name);

    struct sample_t {
        positioned_property<std::string> s_line;
        std::string s_description;
        log_level_t s_level{LEVEL_UNKNOWN};
        std::set<std::string> s_matched_regexes;
    };

    struct action_def {
        std::string ad_name;
        std::string ad_label;
        std::vector<std::string> ad_cmdline;
        bool ad_capture_output{false};

        bool operator<(const action_def& rhs) const
        {
            return this->ad_name < rhs.ad_name;
        }
    };

    virtual ~log_format() = default;

    virtual void clear()
    {
        this->lf_pattern_locks.clear();
        this->lf_date_time.clear();
        this->lf_time_scanner.clear();
    }

    /**
     * Get the name of this log format.
     *
     * @return The log format name.
     */
    virtual const intern_string_t get_name() const = 0;

    struct name_matched {};
    struct name_mismatched {
        size_t nm_partial;
        std::string nm_pattern;
    };

    using match_name_result
        = mapbox::util::variant<name_matched, name_mismatched>;

    virtual match_name_result match_name(const std::string& filename)
    {
        return name_matched{};
    }

    struct scan_match {
        uint32_t sm_quality{0};
        uint32_t sm_strikes{0};
    };

    struct scan_no_match {
        const char* snm_reason{nullptr};
    };

    struct scan_incomplete {};

    using scan_result_t
        = mapbox::util::variant<scan_match, scan_no_match, scan_incomplete>;

    virtual scan_result_t test_line(
        sample_t& sample, std::vector<lnav::console::user_message>& msgs);

    /**
     * Scan a log line to see if it matches this log format.
     *
     * @param dst The vector of loglines that the formatter should append to
     *   if it detected a match.
     * @param offset The offset in the file where this line is located.
     * @param prefix The contents of the line.
     * @param len The length of the prefix string.
     */
    virtual scan_result_t scan(logfile& lf,
                               std::vector<logline>& dst,
                               const line_info& li,
                               shared_buffer_ref& sbr,
                               scan_batch_context& sbc)
        = 0;

    virtual bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const
    {
        return false;
    }

    /**
     * Remove redundant data from the log line string.
     *
     * XXX We should probably also add some attributes to the line here, so we
     * can highlight things like the date.
     *
     * @param line The log line to edit.
     */
    virtual void scrub(std::string& line) {}

    virtual void annotate(logfile* lf,
                          uint64_t line_number,
                          string_attrs_t& sa,
                          logline_value_vector& values,
                          bool annotate_module = true) const;

    virtual void rewrite(exec_context& ec,
                         shared_buffer_ref& line,
                         string_attrs_t& sa,
                         std::string& value_out)
    {
        value_out.assign(line.get_data(), line.length());
    }

    virtual const logline_value_stats* stats_for_value(
        const intern_string_t& name) const
    {
        return nullptr;
    }

    virtual std::shared_ptr<log_format> specialized(int fmt_lock = -1) = 0;

    virtual std::shared_ptr<log_vtab_impl> get_vtab_impl() const
    {
        return nullptr;
    }

    virtual void get_subline(const logline& ll,
                             shared_buffer_ref& sbr,
                             subline_options opts = subline_options{})
    {
    }

    virtual const std::vector<std::string>* get_actions(
        const logline_value& lv) const
    {
        return nullptr;
    }

    virtual std::set<std::string> get_source_path() const
    {
        std::set<std::string> retval;

        retval.insert("default");

        return retval;
    }

    virtual bool hide_field(const intern_string_t field_name, bool val)
    {
        return false;
    }

    virtual std::map<intern_string_t, logline_value_meta> get_field_states()
    {
        return {};
    }

    const char* const* get_timestamp_formats() const
    {
        if (this->lf_timestamp_format.empty()) {
            return nullptr;
        }

        return &this->lf_timestamp_format[0];
    }

    void check_for_new_year(std::vector<logline>& dst,
                            exttm log_tv,
                            timeval timeval1);

    virtual std::string get_pattern_path(uint64_t line_number) const;

    virtual intern_string_t get_pattern_name(uint64_t line_number) const;

    virtual std::string get_pattern_regex(uint64_t line_number) const
    {
        return "";
    }

    virtual std::vector<logline_value_meta> get_value_metadata() const
    {
        return {};
    }

    virtual bool format_changed() { return false; }

    struct pattern_for_lines {
        pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index);

        uint32_t pfl_line;
        int pfl_pat_index;
    };

    int last_pattern_index() const
    {
        if (this->lf_pattern_locks.empty()) {
            return -1;
        }

        return this->lf_pattern_locks.back().pfl_pat_index;
    }

    int pattern_index_for_line(uint64_t line_number) const;

    bool operator<(const log_format& rhs) const
    {
        return this->get_name() < rhs.get_name();
    }

    static bool name_lt(const std::shared_ptr<const log_format>& lhs,
                        const std::shared_ptr<const log_format>& rhs)
    {
        return intern_string_t::case_lt(lhs->get_name(), rhs->get_name());
    }

    exttm tm_for_display(logfile::iterator ll, string_fragment sf);

    enum class subsecond_unit {
        milli,
        micro,
        nano,
    };

    std::string lf_description;
    log_format* lf_root_format{this};
    uint8_t lf_mod_index{0};
    bool lf_multiline{true};
    bool lf_structured{false};
    bool lf_formatted_lines{false};
    date_time_scanner lf_date_time;
    date_time_scanner lf_time_scanner;
    std::vector<pattern_for_lines> lf_pattern_locks;
    intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)};
    intern_string_t lf_subsecond_field;
    std::optional<subsecond_unit> lf_subsecond_unit;
    intern_string_t lf_time_field;
    std::vector<const char*> lf_timestamp_format;
    unsigned int lf_timestamp_flags{0};
    std::map<std::string, action_def> lf_action_defs;
    std::vector<logline_value_stats> lf_value_stats;
    std::vector<highlighter> lf_highlighters;
    bool lf_is_self_describing{false};
    bool lf_time_ordered{true};
    bool lf_specialized{false};
    bool lf_level_hideable{true};
    std::optional<uint64_t> lf_max_unrecognized_lines;
    std::map<const intern_string_t, std::shared_ptr<format_tag_def>>
        lf_tag_defs;

    std::map<const intern_string_t, std::shared_ptr<format_partition_def>>
        lf_partition_defs;

    struct opid_descriptor {
        positioned_property<intern_string_t> od_field;
        factory_container<lnav::pcre2pp::code> od_extractor;
        std::string od_prefix{" "};
        std::string od_suffix;
        std::string od_joiner{", "};

        std::optional<std::string> matches(const string_fragment& sf) const;
    };

    struct opid_descriptors {
        std::shared_ptr<std::vector<opid_descriptor>> od_descriptors;

        std::string to_string(
            const lnav::map::small<size_t, std::string>& lod) const;
    };

    std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
        lf_opid_description_def{
            std::make_shared<std::map<intern_string_t, opid_descriptors>>()};

    std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
        lf_subid_description_def{
            std::make_shared<std::map<intern_string_t, opid_descriptors>>()};

    ArenaAlloc::Alloc<char> lf_desc_allocator{2 * 1024};

    using desc_field_set
        = robin_hood::unordered_set<intern_string_t,
                                    intern_hasher,
                                    std::equal_to<intern_string_t>>;

    desc_field_set lf_desc_fields;

    using desc_cap_map
        = robin_hood::unordered_map<intern_string_t,
                                    string_fragment,
                                    intern_hasher,
                                    std::equal_to<intern_string_t>>;
    desc_cap_map lf_desc_captures;

    static const intern_string_t LOG_TIME_STR;
    static const intern_string_t LOG_LEVEL_STR;
    static const intern_string_t LOG_OPID_STR;

protected:
    static std::vector<std::shared_ptr<log_format>> lf_root_formats;

    struct pcre_format {
        template<typename T, std::size_t N>
        explicit pcre_format(const T (&regex)[N])
            : name(regex),
              pcre(lnav::pcre2pp::code::from_const(regex).to_shared()),
              pf_timestamp_index(this->pcre->name_index("timestamp"))
        {
        }

        pcre_format() = default;

        const char* name{nullptr};
        std::shared_ptr<lnav::pcre2pp::code> pcre;
        int pf_timestamp_index{-1};
    };

    static bool next_format(const pcre_format* fmt,
                            int& index,
                            int& locked_index);

    const char* log_scanf(uint32_t line_number,
                          string_fragment line,
                          const pcre_format* fmt,
                          const char* time_fmt[],
                          struct exttm* tm_out,
                          struct timeval* tv_out,

                          string_fragment* ts_out,
                          std::optional<string_fragment>* level_out);
};

#endif

1	/**
2	* Copyright (c) 2007-2012, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file log_format.hh
30	*/
31
32	#ifndef log_format_hh
33	#define log_format_hh
34
35	#include <stdint.h>
36	#include <sys/time.h>
37	#include <time.h>
38	#define __STDC_FORMAT_MACROS
39	#include <limits>
40	#include <memory>
41	#include <set>
42	#include <string>
43	#include <utility>
44	#include <vector>
45
46	#include <inttypes.h>
47	#include <sys/types.h>
48
49	#include "base/date_time_scanner.hh"
50	#include "base/intern_string.hh"
51	#include "base/lnav_log.hh"
52	#include "highlighter.hh"
53	#include "line_buffer.hh"
54	#include "log_format_fwd.hh"
55	#include "logfile.hh"
56	#include "pcrepp/pcre2pp.hh"
57	#include "robin_hood/robin_hood.h"
58	#include "shared_buffer.hh"
59
60	struct sqlite3;
61	class logfile;
62	class log_vtab_manager;
63	struct exec_context;
64
65	enum class scale_op_t {
66	SO_IDENTITY,
67	SO_MULTIPLY,
68	SO_DIVIDE
69	};
70
71	struct scaling_factor {
72	template<typename T>
73	void scale(T& val) const	90✔
74	{
75	switch (this->sf_op) {	90✔
UNCOV 76	case scale_op_t::SO_IDENTITY:	×
UNCOV 77	break;	×
78	case scale_op_t::SO_DIVIDE:	90✔
79	val = val / (T) this->sf_value;	90✔
80	break;	90✔
UNCOV 81	case scale_op_t::SO_MULTIPLY:	×
UNCOV 82	val = val * (T) this->sf_value;	×
83	break;	×
84	}
85	}	90✔
86
87	scale_op_t sf_op{scale_op_t::SO_IDENTITY};
88	double sf_value{1};
89	};
90
91	enum class value_kind_t : int {
92	VALUE_UNKNOWN = -1,
93	VALUE_NULL,
94	VALUE_TEXT,
95	VALUE_INTEGER,
96	VALUE_FLOAT,
97	VALUE_BOOLEAN,
98	VALUE_JSON,
99	VALUE_STRUCT,
100	VALUE_QUOTED,
101	VALUE_W3C_QUOTED,
102	VALUE_TIMESTAMP,
103	VALUE_XML,
104
105	VALUE__MAX
106	};
107
108	enum class chart_type_t {
109	none,
110	hist,
111	spectro,
112	};
113
114	struct logline_value_meta {
115	struct internal_column {
UNCOV 116	bool operator==(const internal_column&) const { return true; }	×
117	};
118	struct external_column {
UNCOV 119	bool operator==(const external_column&) const { return true; }	×
120	};
121	struct table_column {
122	size_t value;
123
124	bool operator==(const table_column& rhs) const	250,836✔
125	{
126	return this->value == rhs.value;	250,836✔
127	}
128	};
129
130	using column_t
131	= mapbox::util::variant<internal_column, external_column, table_column>;
132
133	logline_value_meta(intern_string_t name,	604,509✔
134	value_kind_t kind,
135	column_t col = external_column{},
136	const std::optional<log_format*>& format = std::nullopt)
137	: lvm_name(name), lvm_kind(kind), lvm_column(col), lvm_format(format)	604,509✔
138	{
139	}	604,509✔
140
141	bool is_hidden() const	19,820✔
142	{
143	if (this->lvm_user_hidden) {	19,820✔
144	return this->lvm_user_hidden.value();	57✔
145	}
146	return this->lvm_hidden;	19,763✔
147	}
148
UNCOV 149	bool is_numeric() const	×
150	{
UNCOV 151	if (this->lvm_identifier \|\| this->lvm_foreign_key) {	×
UNCOV 152	return false;	×
153	}
UNCOV 154	switch (this->lvm_kind) {	×
UNCOV 155	case value_kind_t::VALUE_FLOAT:	×
156	case value_kind_t::VALUE_INTEGER:
UNCOV 157	return true;	×
UNCOV 158	default:	×
UNCOV 159	return false;	×
160	}
161	}
162
163	logline_value_meta& with_struct_name(intern_string_t name)	99✔
164	{
165	this->lvm_struct_name = name;	99✔
166	return *this;	99✔
167	}
168
169	chart_type_t to_chart_type() const;
170
171	intern_string_t lvm_name;
172	value_kind_t lvm_kind;
173	column_t lvm_column{external_column{}};
174	std::optional<size_t> lvm_values_index;
175	bool lvm_identifier{false};
176	bool lvm_foreign_key{false};
177	bool lvm_hidden{false};
178	std::optional<bool> lvm_user_hidden;
179	bool lvm_from_module{false};
180	intern_string_t lvm_struct_name;
181	std::optional<log_format*> lvm_format;
182	};
183
184	class logline_value {
185	public:
186	logline_value(logline_value_meta lvm) : lv_meta(std::move(lvm))	78,568✔
187	{
188	this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL;	78,568✔
189	}	78,568✔
190
191	logline_value(logline_value_meta lvm, bool b)	262✔
192	: lv_meta(std::move(lvm)), lv_value((int64_t) (b ? 1 : 0))	262✔
193	{
194	this->lv_meta.lvm_kind = value_kind_t::VALUE_BOOLEAN;	262✔
195	}	262✔
196
197	logline_value(logline_value_meta lvm, int64_t i)	790✔
198	: lv_meta(std::move(lvm)), lv_value(i)	790✔
199	{
200	this->lv_meta.lvm_kind = value_kind_t::VALUE_INTEGER;	790✔
201	}	790✔
202
203	logline_value(logline_value_meta lvm, double i)	14✔
204	: lv_meta(std::move(lvm)), lv_value(i)	14✔
205	{
206	this->lv_meta.lvm_kind = value_kind_t::VALUE_FLOAT;	14✔
207	}	14✔
208
209	logline_value(logline_value_meta lvm, string_fragment frag)	6,677✔
210	: lv_meta(std::move(lvm)), lv_frag(frag)	6,677✔
211	{
212	}	6,677✔
213
214	logline_value(logline_value_meta lvm, const intern_string_t val)
215	: lv_meta(std::move(lvm)), lv_intern_string(val)
216	{
217	}
218
219	logline_value(logline_value_meta lvm, std::string val)	2,371✔
220	: lv_meta(std::move(lvm)), lv_str(std::move(val))	2,371✔
221	{
222	}	2,371✔
223
224	logline_value(logline_value_meta lvm,
225	shared_buffer_ref& sbr,
226	struct line_range origin);
227
228	void apply_scaling(const scaling_factor* sf)	41,870✔
229	{
230	if (sf != nullptr) {	41,870✔
231	switch (this->lv_meta.lvm_kind) {	10✔
UNCOV 232	case value_kind_t::VALUE_INTEGER:	×
233	sf->scale(this->lv_value.i);	×
UNCOV 234	break;	×
235	case value_kind_t::VALUE_FLOAT:	10✔
236	sf->scale(this->lv_value.d);	10✔
237	break;	10✔
UNCOV 238	default:	×
UNCOV 239	break;	×
240	}
241	}
242	}	41,870✔
243
244	std::string to_string() const;
245
246	string_fragment to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const;
247
248	const char* text_value() const	68,282✔
249	{
250	if (this->lv_str) {	68,282✔
251	return this->lv_str->c_str();	114✔
252	}
253	if (this->lv_frag.empty()) {	68,168✔
254	if (this->lv_intern_string.empty()) {	8✔
255	return "";	8✔
256	}
UNCOV 257	return this->lv_intern_string.get();	×
258	}
259	return this->lv_frag.data();	68,160✔
260	}
261
262	size_t text_length() const	68,315✔
263	{
264	if (this->lv_str) {	68,315✔
265	return this->lv_str->size();	114✔
266	}
267	if (this->lv_frag.empty()) {	68,201✔
268	return this->lv_intern_string.size();	8✔
269	}
270	return this->lv_frag.length();	68,193✔
271	}
272
273	struct line_range origin_in_full_msg(const char* msg, ssize_t len) const;
274
275	logline_value_meta lv_meta;
276	union value_u {
277	int64_t i;
278	double d;
279
280	value_u() : i(0) {}	697,792✔
281	value_u(int64_t i) : i(i) {}	1,052✔
282	value_u(double d) : d(d) {}	14✔
283	} lv_value;
284	std::optional<std::string> lv_str;
285	string_fragment lv_frag;
286	int lv_sub_offset{0};
287	intern_string_t lv_intern_string;
288	struct line_range lv_origin;
289	};
290
291	struct logline_value_vector {
292	enum class opid_provenance {
293	none,
294	file,
295	user,
296	};
297
298	void clear()	38,597✔
299	{
300	this->lvv_values.clear();	38,597✔
301	this->lvv_sbr.disown();	38,597✔
302	this->lvv_opid_value = std::nullopt;	38,597✔
303	this->lvv_opid_provenance = opid_provenance::none;	38,597✔
304	}	38,597✔
305
306	logline_value_vector() = default;	51,893✔
307
308	logline_value_vector(const logline_value_vector& other)	720✔
309	: lvv_sbr(other.lvv_sbr.clone()), lvv_values(other.lvv_values),	720✔
310	lvv_opid_value(other.lvv_opid_value),	720✔
311	lvv_opid_provenance(other.lvv_opid_provenance)	720✔
312	{
313	}	720✔
314
315	logline_value_vector& operator=(const logline_value_vector& other)	207✔
316	{
317	this->lvv_sbr = other.lvv_sbr.clone();	207✔
318	this->lvv_values = other.lvv_values;	207✔
319	this->lvv_opid_value = other.lvv_opid_value;	207✔
320	this->lvv_opid_provenance = other.lvv_opid_provenance;	207✔
321
322	return *this;	207✔
323	}
324
325	shared_buffer_ref lvv_sbr;
326	std::vector<logline_value> lvv_values;
327	std::optional<std::string> lvv_opid_value;
328	opid_provenance lvv_opid_provenance{opid_provenance::none};
329	};
330
331	struct logline_value_stats {
332	logline_value_stats() { this->clear(); }	632,248✔
333
334	void clear()	632,285✔
335	{
336	this->lvs_width = 0;	632,285✔
337	this->lvs_count = 0;	632,285✔
338	this->lvs_total = 0;	632,285✔
339	this->lvs_min_value = std::numeric_limits<double>::max();	632,285✔
340	this->lvs_max_value = -std::numeric_limits<double>::max();	632,285✔
341	}	632,285✔
342
343	void merge(const logline_value_stats& other);
344
345	void add_value(double value);
346
347	int64_t lvs_width;
348	int64_t lvs_count;
349	double lvs_total;
350	double lvs_min_value;
351	double lvs_max_value;
352	};
353
354	struct logline_value_name_cmp {
355	explicit logline_value_name_cmp(const intern_string_t* name)	11,288✔
356	: lvc_name(name)	11,288✔
357	{
358	}	11,288✔
359
360	bool operator()(const logline_value& lv) const	73,059✔
361	{
362	return (*this->lvc_name) == lv.lv_meta.lvm_name;	73,059✔
363	}
364
365	const intern_string_t* lvc_name;
366	};
367
368	struct logline_value_col_eq {
369	explicit logline_value_col_eq(const logline_value_meta::table_column& col)	76,655✔
370	: lvc_column(col)	76,655✔
371	{
372	}	76,655✔
373
374	bool operator()(const logline_value& lv) const	265,313✔
375	{
376	if (lv.lv_meta.lvm_column.is<logline_value_meta::table_column>()) {	265,313✔
377	return this->lvc_column	250,770✔
378	== lv.lv_meta.lvm_column	250,770✔
379	.get<logline_value_meta::table_column>();	250,770✔
380	}
381	return false;	14,543✔
382	}
383
384	const logline_value_meta::table_column& lvc_column;
385	};
386
387	class log_vtab_impl;
388
389	/**
390	* Base class for implementations of log format parsers.
391	*/
392	class log_format {
393	public:
394	/**
395	* @return The collection of builtin log formats.
396	*/
397	static std::vector<std::shared_ptr<log_format>>& get_root_formats();
398
399	static std::shared_ptr<log_format> find_root_format(const char* name);
400
401	struct sample_t {
402	positioned_property<std::string> s_line;
403	std::string s_description;
404	log_level_t s_level{LEVEL_UNKNOWN};
405	std::set<std::string> s_matched_regexes;
406	};
407
408	struct action_def {
409	std::string ad_name;
410	std::string ad_label;
411	std::vector<std::string> ad_cmdline;
412	bool ad_capture_output{false};
413
414	bool operator<(const action_def& rhs) const
415	{
416	return this->ad_name < rhs.ad_name;
417	}
418	};
419
420	virtual ~log_format() = default;	50,876✔
421
422	virtual void clear()	737,773✔
423	{
424	this->lf_pattern_locks.clear();	737,773✔
425	this->lf_date_time.clear();	737,773✔
426	this->lf_time_scanner.clear();	737,773✔
427	}	737,773✔
428
429	/**
430	* Get the name of this log format.
431	*
432	* @return The log format name.
433	*/
434	virtual const intern_string_t get_name() const = 0;
435
436	struct name_matched {};
437	struct name_mismatched {
438	size_t nm_partial;
439	std::string nm_pattern;
440	};
441
442	using match_name_result
443	= mapbox::util::variant<name_matched, name_mismatched>;
444
445	virtual match_name_result match_name(const std::string& filename)	51,890✔
446	{
447	return name_matched{};	51,890✔
448	}
449
450	struct scan_match {
451	uint32_t sm_quality{0};
452	uint32_t sm_strikes{0};
453	};
454
455	struct scan_no_match {
456	const char* snm_reason{nullptr};
457	};
458
459	struct scan_incomplete {};
460
461	using scan_result_t
462	= mapbox::util::variant<scan_match, scan_no_match, scan_incomplete>;
463
464	virtual scan_result_t test_line(
465	sample_t& sample, std::vector<lnav::console::user_message>& msgs);
466
467	/**
468	* Scan a log line to see if it matches this log format.
469	*
470	* @param dst The vector of loglines that the formatter should append to
471	* if it detected a match.
472	* @param offset The offset in the file where this line is located.
473	* @param prefix The contents of the line.
474	* @param len The length of the prefix string.
475	*/
476	virtual scan_result_t scan(logfile& lf,
477	std::vector<logline>& dst,
478	const line_info& li,
479	shared_buffer_ref& sbr,
480	scan_batch_context& sbc)
481	= 0;
482
UNCOV 483	virtual bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const	×
484	{
UNCOV 485	return false;	×
486	}
487
488	/**
489	* Remove redundant data from the log line string.
490	*
491	* XXX We should probably also add some attributes to the line here, so we
492	* can highlight things like the date.
493	*
494	* @param line The log line to edit.
495	*/
UNCOV 496	virtual void scrub(std::string& line) {}	×
497
498	virtual void annotate(logfile* lf,
499	uint64_t line_number,
500	string_attrs_t& sa,
501	logline_value_vector& values,
502	bool annotate_module = true) const;
503
504	virtual void rewrite(exec_context& ec,	5✔
505	shared_buffer_ref& line,
506	string_attrs_t& sa,
507	std::string& value_out)
508	{
509	value_out.assign(line.get_data(), line.length());	5✔
510	}	5✔
511
512	virtual const logline_value_stats* stats_for_value(	×
513	const intern_string_t& name) const
514	{
UNCOV 515	return nullptr;	×
516	}
517
518	virtual std::shared_ptr<log_format> specialized(int fmt_lock = -1) = 0;
519
520	virtual std::shared_ptr<log_vtab_impl> get_vtab_impl() const	1,158✔
521	{
522	return nullptr;	1,158✔
523	}
524
525	virtual void get_subline(const logline& ll,	532✔
526	shared_buffer_ref& sbr,
527	subline_options opts = subline_options{})
528	{
529	}	532✔
530
UNCOV 531	virtual const std::vector<std::string>* get_actions(	×
532	const logline_value& lv) const
533	{
UNCOV 534	return nullptr;	×
535	}
536
537	virtual std::set<std::string> get_source_path() const	1✔
538	{
539	std::set<std::string> retval;	1✔
540
541	retval.insert("default");	1✔
542
543	return retval;	1✔
UNCOV 544	}	×
545
UNCOV 546	virtual bool hide_field(const intern_string_t field_name, bool val)	×
547	{
UNCOV 548	return false;	×
549	}
550
551	virtual std::map<intern_string_t, logline_value_meta> get_field_states()	238✔
552	{
553	return {};	238✔
554	}
555
556	const char* const* get_timestamp_formats() const	173,712✔
557	{
558	if (this->lf_timestamp_format.empty()) {	173,712✔
559	return nullptr;	153,220✔
560	}
561
562	return &this->lf_timestamp_format[0];	20,492✔
563	}
564
565	void check_for_new_year(std::vector<logline>& dst,
566	exttm log_tv,
567	timeval timeval1);
568
569	virtual std::string get_pattern_path(uint64_t line_number) const;
570
571	virtual intern_string_t get_pattern_name(uint64_t line_number) const;
572
UNCOV 573	virtual std::string get_pattern_regex(uint64_t line_number) const	×
574	{
575	return "";	×
576	}
577
UNCOV 578	virtual std::vector<logline_value_meta> get_value_metadata() const	×
579	{
UNCOV 580	return {};	×
581	}
582
583	virtual bool format_changed() { return false; }	432✔
584
585	struct pattern_for_lines {
586	pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index);
587
588	uint32_t pfl_line;
589	int pfl_pat_index;
590	};
591
592	int last_pattern_index() const	592,770✔
593	{
594	if (this->lf_pattern_locks.empty()) {	592,770✔
595	return -1;	585,546✔
596	}
597
598	return this->lf_pattern_locks.back().pfl_pat_index;	7,224✔
599	}
600
601	int pattern_index_for_line(uint64_t line_number) const;
602
603	bool operator<(const log_format& rhs) const
604	{
605	return this->get_name() < rhs.get_name();
606	}
607
608	static bool name_lt(const std::shared_ptr<const log_format>& lhs,
609	const std::shared_ptr<const log_format>& rhs)
610	{
611	return intern_string_t::case_lt(lhs->get_name(), rhs->get_name());
612	}
613
614	exttm tm_for_display(logfile::iterator ll, string_fragment sf);
615
616	enum class subsecond_unit {
617	milli,
618	micro,
619	nano,
620	};
621
622	std::string lf_description;
623	log_format* lf_root_format{this};
624	uint8_t lf_mod_index{0};
625	bool lf_multiline{true};
626	bool lf_structured{false};
627	bool lf_formatted_lines{false};
628	date_time_scanner lf_date_time;
629	date_time_scanner lf_time_scanner;
630	std::vector<pattern_for_lines> lf_pattern_locks;
631	intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)};
632	intern_string_t lf_subsecond_field;
633	std::optional<subsecond_unit> lf_subsecond_unit;
634	intern_string_t lf_time_field;
635	std::vector<const char*> lf_timestamp_format;
636	unsigned int lf_timestamp_flags{0};
637	std::map<std::string, action_def> lf_action_defs;
638	std::vector<logline_value_stats> lf_value_stats;
639	std::vector<highlighter> lf_highlighters;
640	bool lf_is_self_describing{false};
641	bool lf_time_ordered{true};
642	bool lf_specialized{false};
643	bool lf_level_hideable{true};
644	std::optional<uint64_t> lf_max_unrecognized_lines;
645	std::map<const intern_string_t, std::shared_ptr<format_tag_def>>
646	lf_tag_defs;
647
648	std::map<const intern_string_t, std::shared_ptr<format_partition_def>>
649	lf_partition_defs;
650
651	struct opid_descriptor {
652	positioned_property<intern_string_t> od_field;
653	factory_container<lnav::pcre2pp::code> od_extractor;
654	std::string od_prefix{" "};
655	std::string od_suffix;
656	std::string od_joiner{", "};
657
658	std::optional<std::string> matches(const string_fragment& sf) const;
659	};
660
661	struct opid_descriptors {
662	std::shared_ptr<std::vector<opid_descriptor>> od_descriptors;
663
664	std::string to_string(
665	const lnav::map::small<size_t, std::string>& lod) const;
666	};
667
668	std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
669	lf_opid_description_def{
670	std::make_shared<std::map<intern_string_t, opid_descriptors>>()};
671
672	std::shared_ptr<std::map<intern_string_t, opid_descriptors>>
673	lf_subid_description_def{
674	std::make_shared<std::map<intern_string_t, opid_descriptors>>()};
675
676	ArenaAlloc::Alloc<char> lf_desc_allocator{2 * 1024};
677
678	using desc_field_set
679	= robin_hood::unordered_set<intern_string_t,
680	intern_hasher,
681	std::equal_to<intern_string_t>>;
682
683	desc_field_set lf_desc_fields;
684
685	using desc_cap_map
686	= robin_hood::unordered_map<intern_string_t,
687	string_fragment,
688	intern_hasher,
689	std::equal_to<intern_string_t>>;
690	desc_cap_map lf_desc_captures;
691
692	static const intern_string_t LOG_TIME_STR;
693	static const intern_string_t LOG_LEVEL_STR;
694	static const intern_string_t LOG_OPID_STR;
695
696	protected:
697	static std::vector<std::shared_ptr<log_format>> lf_root_formats;
698
699	struct pcre_format {
700	template<typename T, std::size_t N>
701	explicit pcre_format(const T (&regex)[N])	6,981✔
702	: name(regex),	6,981✔
703	pcre(lnav::pcre2pp::code::from_const(regex).to_shared()),	6,981✔
704	pf_timestamp_index(this->pcre->name_index("timestamp"))	6,981✔
705	{
706	}	6,981✔
707
708	pcre_format() = default;
709
710	const char* name{nullptr};
711	std::shared_ptr<lnav::pcre2pp::code> pcre;
712	int pf_timestamp_index{-1};
713	};
714
715	static bool next_format(const pcre_format* fmt,
716	int& index,
717	int& locked_index);
718
719	const char* log_scanf(uint32_t line_number,
720	string_fragment line,
721	const pcre_format* fmt,
722	const char* time_fmt[],
723	struct exttm* tm_out,
724	struct timeval* tv_out,
725
726	string_fragment* ts_out,
727	std::optional<string_fragment>* level_out);
728	};
729
730	#endif

tstack / lnav / 17589970077-2502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous