18230413998-2546

Committed 03 Oct 2025 06:20PM UTC coverage: 64.943% (+0.1%) from 64.848%

Build # 18230413998-2546

Build Type

push

github

Committed by

tstack

Commit Message

[log2src] plumb stack trace

... and a pile of minor fixes

Coverage Stats

41 of 52 new or added lines in 11 files covered. (78.85%)

2 existing lines in 2 files now uncovered.

46070 of 70939 relevant lines covered (64.94%)

406890.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.58

/src/all_logs_vtab.cc

/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "all_logs_vtab.hh"

#include "base/attr_line.hh"
#include "base/intern_string.hh"
#include "config.h"
#include "data_parser.hh"
#include "elem_to_json.hh"
#include "hasher.hh"
#include "scn/scan.h"

#ifdef HAVE_RUST_DEPS
#    include "lnav_rs_ext.cxx.hh"
#endif

static auto intern_lifetime = intern_string::get_table_lifetime();

all_logs_vtab::all_logs_vtab()
    : log_vtab_impl(intern_string::lookup("all_logs")),
      alv_msg_meta(intern_string::lookup("log_msg_format"),
                   value_kind_t::VALUE_TEXT,
                   logline_value_meta::table_column{0}),
      alv_schema_meta(intern_string::lookup("log_msg_schema"),
                      value_kind_t::VALUE_TEXT,
                      logline_value_meta::table_column{1}),
      alv_values_meta(intern_string::lookup("log_msg_values"),
                      value_kind_t::VALUE_JSON,
                      logline_value_meta::table_column{2}),
      alv_src_meta(intern_string::lookup("log_msg_src"),
                   value_kind_t::VALUE_JSON,
                   logline_value_meta::table_column{3}),
      alv_thread_meta(intern_string::lookup("log_thread_id"),
                      value_kind_t::VALUE_TEXT,
                      logline_value_meta::table_column{4}),
      alv_exception_meta(intern_string::lookup("log_exception_trace"),
                         value_kind_t::VALUE_TEXT,
                         logline_value_meta::table_column{5})
{
    this->alv_msg_meta.lvm_identifier = true;
    this->alv_schema_meta.lvm_identifier = true;
}

void
all_logs_vtab::get_columns(std::vector<vtab_column>& cols) const
{
    cols.emplace_back(
        vtab_column(this->alv_msg_meta.lvm_name.get())
            .with_comment(
                "The message format with variables replaced by hash marks"));
    cols.emplace_back(this->alv_schema_meta.lvm_name.get(),
                      SQLITE3_TEXT,
                      "",
                      true,
                      "The ID for the message schema");
    cols.emplace_back(this->alv_values_meta.lvm_name.get(),
                      SQLITE3_TEXT,
                      "",
                      false,
                      "The values extracted from the message");
    cols.emplace_back(this->alv_src_meta.lvm_name.get(),
                      SQLITE3_TEXT,
                      "",
                      false,
                      "The source code that generated this message");
    cols.emplace_back(this->alv_thread_meta.lvm_name.get(),
                      SQLITE3_TEXT,
                      "",
                      false,
                      "The ID of the thread that generated this message");
    cols.emplace_back(this->alv_exception_meta.lvm_name.get(),
                      SQLITE3_TEXT,
                      "",
                      false,
                      "If there is an exception, the stack trace details");
}

void
all_logs_vtab::extract(logfile* lf,
                       uint64_t line_number,
                       string_attrs_t& sa,
                       logline_value_vector& values)
{
    auto& line = values.lvv_sbr;
    auto* format = lf->get_format_ptr();

    logline_value_vector sub_values;

    sa.clear();
    sub_values.lvv_sbr = line.clone();
    format->annotate(lf, line_number, sa, sub_values, false);

    auto body = find_string_attr_range(sa, &SA_BODY);
    if (!body.is_valid()) {
        body.lr_start = 0;
        body.lr_end = line.length();
    }
    auto body_sf = line.to_string_fragment(body);
    auto src_file = find_string_attr_range(sa, &SA_SRC_FILE);
    auto src_line = find_string_attr_range(sa, &SA_SRC_LINE);
    auto thread_id = find_string_attr_range(sa, &SA_THREAD_ID);
    auto src_file_sf = line.to_string_fragment(src_file);
    auto src_line_sf = line.to_string_fragment(src_line);
    auto thread_id_sf = line.to_string_fragment(thread_id);
    auto h = hasher();
    if (src_file_sf.is_valid() && src_line_sf.is_valid()) {
        h.update(format->get_name().c_str());
        h.update(src_file_sf);
        h.update(src_line_sf);
    }
#ifdef HAVE_RUST_DEPS
    auto file_rust_str = rust::Str();
    auto lineno = 0UL;
    if (src_file_sf.is_valid()) {
        file_rust_str = rust::Str(src_file_sf.data(), src_file_sf.length());
    }
    if (src_line_sf.is_valid()) {
        auto scan_res
            = scn::scan_int<decltype(lineno)>(src_line_sf.to_string_view());
        if (scan_res) {
            lineno = scan_res->value();
        }
    }
    auto body_rust_str = rust::Str(body_sf.data(), body_sf.length());
    auto find_res = lnav_rs_ext::find_log_statement_json(
        file_rust_str, lineno, body_rust_str);
    if (find_res != nullptr) {
        if (!src_file_sf.is_valid() || !src_line_sf.is_valid()) {
            h.update(find_res->src.c_str());
            h.update(find_res->pattern.c_str());
        }
        auto line_iter = lf->begin() + line_number;
        if (!line_iter->has_schema()) {
            line_iter->set_schema(h.to_array());
        }
        values.lvv_values.emplace_back(this->alv_msg_meta,
                                       (std::string) find_res->pattern);
        values.lvv_values.emplace_back(this->alv_schema_meta, h.to_string());
        values.lvv_values.emplace_back(this->alv_values_meta,
                                       (std::string) find_res->variables);
        values.lvv_values.emplace_back(this->alv_src_meta,
                                       (std::string) find_res->src);
        values.lvv_values.emplace_back(this->alv_exception_meta,
                                       (std::string) find_res->exception_trace);
    } else
#endif
    {
        data_scanner ds(body_sf);
        data_parser dp(&ds);
        std::string str;

        dp.dp_msg_format = &str;
        dp.parse();

        yajlpp_gen gen;
        yajl_gen_config(gen, yajl_gen_beautify, false);

        elements_to_json(gen, dp, &dp.dp_pairs);

        auto schema_id = (src_file_sf.is_valid() && src_line_sf.is_valid())
            ? h.to_string()
            : dp.dp_schema_id.to_string();
        values.lvv_values.emplace_back(this->alv_msg_meta, std::move(str));
        values.lvv_values.emplace_back(this->alv_schema_meta, schema_id);
        values.lvv_values.emplace_back(
            this->alv_values_meta,
            json_string(gen).to_string_fragment().to_string());
    }
    if (thread_id_sf.empty()) {
        values.lvv_values.emplace_back(this->alv_thread_meta);
    } else {
        values.lvv_values.emplace_back(this->alv_thread_meta, thread_id_sf);
    }
    values.lvv_opid_value = std::move(sub_values.lvv_opid_value);
    values.lvv_opid_provenance = sub_values.lvv_opid_provenance;
}

bool
all_logs_vtab::next(log_cursor& lc, logfile_sub_source& lss)
{
    return true;
}

1	/**
2	* Copyright (c) 2020, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*/
29
30	#include "all_logs_vtab.hh"
31
32	#include "base/attr_line.hh"
33	#include "base/intern_string.hh"
34	#include "config.h"
35	#include "data_parser.hh"
36	#include "elem_to_json.hh"
37	#include "hasher.hh"
38	#include "scn/scan.h"
39
40	#ifdef HAVE_RUST_DEPS
41	# include "lnav_rs_ext.cxx.hh"
42	#endif
43
44	static auto intern_lifetime = intern_string::get_table_lifetime();
45
46	all_logs_vtab::all_logs_vtab()	586✔
47	: log_vtab_impl(intern_string::lookup("all_logs")),
48	alv_msg_meta(intern_string::lookup("log_msg_format"),	1,172✔
49	value_kind_t::VALUE_TEXT,
50	logline_value_meta::table_column{0}),	586✔
51	alv_schema_meta(intern_string::lookup("log_msg_schema"),	1,758✔
52	value_kind_t::VALUE_TEXT,
53	logline_value_meta::table_column{1}),	586✔
54	alv_values_meta(intern_string::lookup("log_msg_values"),	1,758✔
55	value_kind_t::VALUE_JSON,
56	logline_value_meta::table_column{2}),	586✔
57	alv_src_meta(intern_string::lookup("log_msg_src"),	1,758✔
58	value_kind_t::VALUE_JSON,
59	logline_value_meta::table_column{3}),	586✔
60	alv_thread_meta(intern_string::lookup("log_thread_id"),	1,758✔
61	value_kind_t::VALUE_TEXT,
62	logline_value_meta::table_column{4}),	586✔
63	alv_exception_meta(intern_string::lookup("log_exception_trace"),	1,758✔
64	value_kind_t::VALUE_TEXT,
65	logline_value_meta::table_column{5})	2,344✔
66	{
67	this->alv_msg_meta.lvm_identifier = true;	586✔
68	this->alv_schema_meta.lvm_identifier = true;	586✔
69	}	586✔
70
71	void
72	all_logs_vtab::get_columns(std::vector<vtab_column>& cols) const	586✔
73	{
74	cols.emplace_back(	586✔
75	vtab_column(this->alv_msg_meta.lvm_name.get())	3,516✔
76	.with_comment(
77	"The message format with variables replaced by hash marks"));
78	cols.emplace_back(this->alv_schema_meta.lvm_name.get(),	586✔
79	SQLITE3_TEXT,	×
80	"",
81	true,	586✔
82	"The ID for the message schema");
83	cols.emplace_back(this->alv_values_meta.lvm_name.get(),	586✔
84	SQLITE3_TEXT,	×
85	"",
86	false,	586✔
87	"The values extracted from the message");
88	cols.emplace_back(this->alv_src_meta.lvm_name.get(),	586✔
89	SQLITE3_TEXT,	×
90	"",
91	false,	586✔
92	"The source code that generated this message");
93	cols.emplace_back(this->alv_thread_meta.lvm_name.get(),	586✔
94	SQLITE3_TEXT,	×
95	"",
96	false,	586✔
97	"The ID of the thread that generated this message");
98	cols.emplace_back(this->alv_exception_meta.lvm_name.get(),	586✔
NEW 99	SQLITE3_TEXT,	×
100	"",
101	false,	586✔
102	"If there is an exception, the stack trace details");
103	}	586✔
104
105	void
106	all_logs_vtab::extract(logfile* lf,	75✔
107	uint64_t line_number,
108	string_attrs_t& sa,
109	logline_value_vector& values)
110	{
111	auto& line = values.lvv_sbr;	75✔
112	auto* format = lf->get_format_ptr();	75✔
113
114	logline_value_vector sub_values;	75✔
115
116	sa.clear();	75✔
117	sub_values.lvv_sbr = line.clone();	75✔
118	format->annotate(lf, line_number, sa, sub_values, false);	75✔
119
120	auto body = find_string_attr_range(sa, &SA_BODY);	75✔
121	if (!body.is_valid()) {	75✔
122	body.lr_start = 0;	×
123	body.lr_end = line.length();	×
124	}
125	auto body_sf = line.to_string_fragment(body);	75✔
126	auto src_file = find_string_attr_range(sa, &SA_SRC_FILE);	75✔
127	auto src_line = find_string_attr_range(sa, &SA_SRC_LINE);	75✔
128	auto thread_id = find_string_attr_range(sa, &SA_THREAD_ID);	75✔
129	auto src_file_sf = line.to_string_fragment(src_file);	75✔
130	auto src_line_sf = line.to_string_fragment(src_line);	75✔
131	auto thread_id_sf = line.to_string_fragment(thread_id);	75✔
132	auto h = hasher();	75✔
133	if (src_file_sf.is_valid() && src_line_sf.is_valid()) {	75✔
134	h.update(format->get_name().c_str());	22✔
135	h.update(src_file_sf);	22✔
136	h.update(src_line_sf);	22✔
137	}
138	#ifdef HAVE_RUST_DEPS
139	auto file_rust_str = rust::Str();	75✔
140	auto lineno = 0UL;	75✔
141	if (src_file_sf.is_valid()) {	75✔
142	file_rust_str = rust::Str(src_file_sf.data(), src_file_sf.length());	22✔
143	}
144	if (src_line_sf.is_valid()) {	75✔
145	auto scan_res
146	= scn::scan_int<decltype(lineno)>(src_line_sf.to_string_view());	22✔
147	if (scan_res) {	22✔
148	lineno = scan_res->value();	22✔
149	}
150	}
151	auto body_rust_str = rust::Str(body_sf.data(), body_sf.length());	75✔
152	auto find_res = lnav_rs_ext::find_log_statement_json(
153	file_rust_str, lineno, body_rust_str);	75✔
154	if (find_res != nullptr) {	75✔
155	if (!src_file_sf.is_valid() \|\| !src_line_sf.is_valid()) {	5✔
156	h.update(find_res->src.c_str());	10✔
157	h.update(find_res->pattern.c_str());	10✔
158	}
159	auto line_iter = lf->begin() + line_number;	5✔
160	if (!line_iter->has_schema()) {	5✔
161	line_iter->set_schema(h.to_array());	5✔
162	}
163	values.lvv_values.emplace_back(this->alv_msg_meta,	5✔
164	(std::string) find_res->pattern);	10✔
165	values.lvv_values.emplace_back(this->alv_schema_meta, h.to_string());	5✔
166	values.lvv_values.emplace_back(this->alv_values_meta,	5✔
167	(std::string) find_res->variables);	10✔
168	values.lvv_values.emplace_back(this->alv_src_meta,	5✔
169	(std::string) find_res->src);	10✔
170	values.lvv_values.emplace_back(this->alv_exception_meta,	5✔
171	(std::string) find_res->exception_trace);	10✔
172	} else
173	#endif
174	{
175	data_scanner ds(body_sf);	70✔
176	data_parser dp(&ds);	70✔
177	std::string str;	70✔
178
179	dp.dp_msg_format = &str;	70✔
180	dp.parse();	70✔
181
182	yajlpp_gen gen;	70✔
183	yajl_gen_config(gen, yajl_gen_beautify, false);	70✔
184
185	elements_to_json(gen, dp, &dp.dp_pairs);	70✔
186
187	auto schema_id = (src_file_sf.is_valid() && src_line_sf.is_valid())	92✔
188	? h.to_string()	70✔
189	: dp.dp_schema_id.to_string();	70✔
190	values.lvv_values.emplace_back(this->alv_msg_meta, std::move(str));	70✔
191	values.lvv_values.emplace_back(this->alv_schema_meta, schema_id);	70✔
192	values.lvv_values.emplace_back(	140✔
193	this->alv_values_meta,	70✔
194	json_string(gen).to_string_fragment().to_string());	140✔
195	}	70✔
196	if (thread_id_sf.empty()) {	75✔
197	values.lvv_values.emplace_back(this->alv_thread_meta);	36✔
198	} else {
199	values.lvv_values.emplace_back(this->alv_thread_meta, thread_id_sf);	39✔
200	}
201	values.lvv_opid_value = std::move(sub_values.lvv_opid_value);	75✔
202	values.lvv_opid_provenance = sub_values.lvv_opid_provenance;	75✔
203	}	75✔
204
205	bool
206	all_logs_vtab::next(log_cursor& lc, logfile_sub_source& lss)	80✔
207	{
208	return true;	80✔
209	}

tstack / lnav / 18230413998-2546

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous