20284884426-2753

Committed 16 Dec 2025 10:23PM UTC coverage: 68.23% (-0.7%) from 68.903%

Build # 20284884426-2753

Build Type

push

github

Committed by

tstack

Commit Message

[log] show invalid utf hex dump in log view too

Run Details

25 of 25 new or added lines in 2 files covered. (100.0%)

503 existing lines in 33 files now uncovered.

51170 of 74996 relevant lines covered (68.23%)

433797.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.59

/src/log_data_helper.cc

/**
 * Copyright (c) 2021, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file log_data_helper.cc
 */

#include <memory>

#include "log_data_helper.hh"

#include "config.h"
#include "lnav_util.hh"
#include "logfile.hh"
#include "pugixml/pugixml.hpp"
#include "scn/scan.h"
#include "sql_util.hh"
#include "xml_util.hh"

#ifdef HAVE_RUST_DEPS
#    include "lnav_rs_ext.cxx.hh"
#endif

void
log_data_helper::clear()
{
    this->ldh_file = nullptr;
    this->ldh_line_values.lvv_sbr.disown();
    this->ldh_parser.reset();
    this->ldh_scanner.reset();
    this->ldh_namer.reset();
    this->ldh_extra_json.clear();
    this->ldh_json_pairs.clear();
    this->ldh_xml_pairs.clear();
    this->ldh_line_attrs.clear();
}

bool
log_data_helper::load_line(content_line_t line, bool allow_middle)
{
    auto retval = false;

    this->ldh_source_line = this->ldh_line_index = line;

    this->ldh_file = this->ldh_log_source.find(this->ldh_line_index);
    auto ll = this->ldh_file->begin() + this->ldh_line_index;
    this->ldh_y_offset = 0;
    while (allow_middle && ll->is_continued()) {
        --ll;
        this->ldh_y_offset += 1;
    }
    this->ldh_line = ll;
    if (!ll->is_message()) {
        this->ldh_parser.reset();
        this->ldh_scanner.reset();
        this->ldh_namer.reset();
        this->ldh_extra_json.clear();
        this->ldh_json_pairs.clear();
        this->ldh_xml_pairs.clear();
        this->ldh_line_attrs.clear();
        this->ldh_msg_format.clear();
#ifdef HAVE_RUST_DEPS
        this->ldh_src_ref = std::nullopt;
        this->ldh_src_vars.clear();
#endif
    } else {
        auto format = this->ldh_file->get_format();
        auto& sa = this->ldh_line_attrs;

#ifdef HAVE_RUST_DEPS
        this->ldh_src_ref = std::nullopt;
        this->ldh_src_vars.clear();
#endif
        this->ldh_parser.reset();
        this->ldh_scanner.reset();
        this->ldh_namer.reset();
        this->ldh_line_attrs.clear();
        this->ldh_line_values.clear();
        this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr);
        this->ldh_line_values.lvv_sbr.erase_ansi();
        format->annotate(this->ldh_file.get(),
                         this->ldh_line_index,
                         sa,
                         this->ldh_line_values);

        this->ldh_extra_json.clear();
        this->ldh_json_pairs.clear();
        this->ldh_xml_pairs.clear();
        for (auto& ldh_line_value : this->ldh_line_values.lvv_values) {
            if (ldh_line_value.lv_meta.lvm_name == format->lf_timestamp_field) {
                continue;
            }
            if (ldh_line_value.lv_meta.lvm_column
                    .is<logline_value_meta::external_column>())
            {
                stack_buf allocator;
                auto* buf = allocator.allocate(
                    ldh_line_value.lv_meta.lvm_name.size() + 2);

                auto rc = fmt::format_to(
                    buf, FMT_STRING("/{}"), ldh_line_value.lv_meta.lvm_name);
                *rc = '\0';
                this->ldh_extra_json[intern_string::lookup(buf, -1)]
                    = ldh_line_value.to_string();
                continue;
            }

            switch (ldh_line_value.lv_meta.lvm_kind) {
                case value_kind_t::VALUE_JSON: {
                    if (!ldh_line_value.lv_meta.lvm_struct_name.empty()) {
                        continue;
                    }

                    auto parse_res = json_walk_collector::parse_fully(
                        ldh_line_value.text_value_fragment());

                    if (parse_res.isOk()) {
                        this->ldh_json_pairs[ldh_line_value.lv_meta.lvm_name]
                            = parse_res.unwrap();
                    }
                    break;
                }
                case value_kind_t::VALUE_XML: {
                    auto col_name = ldh_line_value.lv_meta.lvm_name;
                    pugi::xml_document doc;

                    auto parse_res
                        = doc.load_buffer(ldh_line_value.text_value(),
                                          ldh_line_value.text_length());

                    if (parse_res) {
                        pugi::xpath_query query("//*");
                        auto node_set = doc.select_nodes(query);

                        for (const auto& xpath_node : node_set) {
                            auto node_path = lnav::pugixml::get_actual_path(
                                xpath_node.node());
                            for (auto& attr : xpath_node.node().attributes()) {
                                auto attr_path
                                    = fmt::format(FMT_STRING("{}/@{}"),
                                                  node_path,
                                                  attr.name());

                                this->ldh_xml_pairs[std::make_pair(col_name,
                                                                   attr_path)]
                                    = attr.value();
                            }

                            if (xpath_node.node().text().empty()) {
                                continue;
                            }

                            auto text_path = fmt::format(
                                FMT_STRING("{}/text()"), node_path);
                            this->ldh_xml_pairs[std::make_pair(col_name,
                                                               text_path)]
                                = trim(xpath_node.node().text().get());
                        }
                    }
                    break;
                }
                default:
                    break;
            }
        }

        retval = true;
    }

    return retval;
}

void
log_data_helper::parse_body()
{
    if (!this->ldh_line->is_message()) {
        return;
    }

    if (!this->ldh_line_values.lvv_sbr.get_metadata().m_valid_utf) {
        return;
    }

    auto& sbr = this->ldh_line_values.lvv_sbr;
    auto& sa = this->ldh_line_attrs;
    auto body = find_string_attr_range(sa, &SA_BODY);
    if (body.lr_start == -1) {
        body.lr_start = this->ldh_line_values.lvv_sbr.length();
        body.lr_end = this->ldh_line_values.lvv_sbr.length();
    }
    auto body_sf = sbr.to_string_fragment(body);
#ifdef HAVE_RUST_DEPS
    auto file_rust_str = rust::Str();
    auto lineno = 0UL;
    auto body_rust_str = rust::Str(body_sf.data(), body_sf.length());
    auto src_file = find_string_attr_range(sa, &SA_SRC_FILE);
    if (src_file.is_valid()) {
        auto src_file_sf = sbr.to_string_fragment(src_file);
        file_rust_str = rust::Str(src_file_sf.data(), src_file_sf.length());
    }
    auto src_line = find_string_attr_range(sa, &SA_SRC_LINE);
    if (src_line.is_valid()) {
        auto src_line_sf = sbr.to_string_fragment(src_line);
        auto scan_res
            = scn::scan_int<decltype(lineno)>(src_line_sf.to_string_view());
        if (scan_res) {
            lineno = scan_res->value();
        }
    }
    this->ldh_src_ref = std::nullopt;
    this->ldh_src_vars.clear();
    auto find_res
        = lnav_rs_ext::find_log_statement(file_rust_str, lineno, body_rust_str);
    if (find_res != nullptr) {
        this->ldh_src_ref = lnav::src_ref{
            std::filesystem::path((std::string) find_res->src.file),
            (uint32_t) find_res->src.begin_line,
            (std::string) find_res->src.name,
        };
        for (const auto& [expr, value] : find_res->variables) {
            this->ldh_src_vars.emplace_back((std::string) expr,
                                            (std::string) value);
        }
    } else
#endif
    {
        this->ldh_scanner = std::make_unique<data_scanner>(body_sf);
        this->ldh_parser
            = std::make_unique<data_parser>(this->ldh_scanner.get());
        this->ldh_msg_format.clear();
        this->ldh_parser->dp_msg_format = &this->ldh_msg_format;
        if (body.length() < 128 * 1024) {
            this->ldh_parser->parse();
        }
        this->ldh_namer
            = std::make_unique<column_namer>(column_namer::language::SQL);
        for (const auto& lv : this->ldh_line_values.lvv_values) {
            this->ldh_namer->cn_builtin_names.emplace_back(
                lv.lv_meta.lvm_name.get());
        }
    }
}

int
log_data_helper::get_line_bounds(size_t& line_index_out,
                                 size_t& line_end_index_out) const
{
    int retval = 0;

    line_end_index_out = 0;
    do {
        line_index_out = line_end_index_out;
        const auto* line_end = (const char*) memchr(
            this->ldh_line_values.lvv_sbr.get_data() + line_index_out + 1,
            '\n',
            this->ldh_line_values.lvv_sbr.length() - line_index_out - 1);
        if (line_end != nullptr) {
            line_end_index_out
                = line_end - this->ldh_line_values.lvv_sbr.get_data();
        } else {
            line_end_index_out = std::string::npos;
        }
        retval += 1;
    } while (retval <= this->ldh_y_offset);

    if (line_end_index_out == std::string::npos) {
        line_end_index_out = this->ldh_line_values.lvv_sbr.length();
    }

    return retval;
}

std::string
log_data_helper::format_json_getter(const intern_string_t field, int index)
{
    std::string retval;

    auto qname = sql_quote_ident(field.get());
    retval = lnav::sql::mprintf(
        "jget(%s,%Q)",
        qname.in(),
        this->ldh_json_pairs[field].jwc_values[index].first.c_str());

    return retval;
}

1	/**
2	* Copyright (c) 2021, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file log_data_helper.cc
30	*/
31
32	#include <memory>
33
34	#include "log_data_helper.hh"
35
36	#include "config.h"
37	#include "lnav_util.hh"
38	#include "logfile.hh"
39	#include "pugixml/pugixml.hpp"
40	#include "scn/scan.h"
41	#include "sql_util.hh"
42	#include "xml_util.hh"
43
44	#ifdef HAVE_RUST_DEPS
45	# include "lnav_rs_ext.cxx.hh"
46	#endif
47
48	void
49	log_data_helper::clear()	×
50	{
51	this->ldh_file = nullptr;	×
52	this->ldh_line_values.lvv_sbr.disown();	×
53	this->ldh_parser.reset();	×
54	this->ldh_scanner.reset();	×
55	this->ldh_namer.reset();	×
56	this->ldh_extra_json.clear();	×
57	this->ldh_json_pairs.clear();	×
58	this->ldh_xml_pairs.clear();	×
59	this->ldh_line_attrs.clear();	×
60	}
61
62	bool
63	log_data_helper::load_line(content_line_t line, bool allow_middle)	22✔
64	{
65	auto retval = false;	22✔
66
67	this->ldh_source_line = this->ldh_line_index = line;	22✔
68
69	this->ldh_file = this->ldh_log_source.find(this->ldh_line_index);	22✔
70	auto ll = this->ldh_file->begin() + this->ldh_line_index;	22✔
71	this->ldh_y_offset = 0;	22✔
72	while (allow_middle && ll->is_continued()) {	23✔
73	--ll;	1✔
74	this->ldh_y_offset += 1;	1✔
75	}
76	this->ldh_line = ll;	22✔
77	if (!ll->is_message()) {	22✔
UNCOV 78	this->ldh_parser.reset();	×
UNCOV 79	this->ldh_scanner.reset();	×
UNCOV 80	this->ldh_namer.reset();	×
UNCOV 81	this->ldh_extra_json.clear();	×
UNCOV 82	this->ldh_json_pairs.clear();	×
UNCOV 83	this->ldh_xml_pairs.clear();	×
UNCOV 84	this->ldh_line_attrs.clear();	×
UNCOV 85	this->ldh_msg_format.clear();	×
86	#ifdef HAVE_RUST_DEPS
UNCOV 87	this->ldh_src_ref = std::nullopt;	×
UNCOV 88	this->ldh_src_vars.clear();	×
89	#endif
90	} else {
91	auto format = this->ldh_file->get_format();	22✔
92	auto& sa = this->ldh_line_attrs;	22✔
93
94	#ifdef HAVE_RUST_DEPS
95	this->ldh_src_ref = std::nullopt;	22✔
96	this->ldh_src_vars.clear();	22✔
97	#endif
98	this->ldh_parser.reset();	22✔
99	this->ldh_scanner.reset();	22✔
100	this->ldh_namer.reset();	22✔
101	this->ldh_line_attrs.clear();	22✔
102	this->ldh_line_values.clear();	22✔
103	this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr);	22✔
104	this->ldh_line_values.lvv_sbr.erase_ansi();	22✔
105	format->annotate(this->ldh_file.get(),	44✔
106	this->ldh_line_index,
107	sa,
108	this->ldh_line_values);	22✔
109
110	this->ldh_extra_json.clear();	22✔
111	this->ldh_json_pairs.clear();	22✔
112	this->ldh_xml_pairs.clear();	22✔
113	for (auto& ldh_line_value : this->ldh_line_values.lvv_values) {	202✔
114	if (ldh_line_value.lv_meta.lvm_name == format->lf_timestamp_field) {	180✔
115	continue;	20✔
116	}
117	if (ldh_line_value.lv_meta.lvm_column	160✔
118	.is<logline_value_meta::external_column>())	160✔
119	{
120	stack_buf allocator;	5✔
121	auto* buf = allocator.allocate(	5✔
122	ldh_line_value.lv_meta.lvm_name.size() + 2);	5✔
123
124	auto rc = fmt::format_to(	×
125	buf, FMT_STRING("/{}"), ldh_line_value.lv_meta.lvm_name);	15✔
126	*rc = '\0';	5✔
127	this->ldh_extra_json[intern_string::lookup(buf, -1)]	5✔
128	= ldh_line_value.to_string();	10✔
129	continue;	5✔
130	}	5✔
131
132	switch (ldh_line_value.lv_meta.lvm_kind) {	155✔
133	case value_kind_t::VALUE_JSON: {	×
134	if (!ldh_line_value.lv_meta.lvm_struct_name.empty()) {	×
135	continue;	×
136	}
137
138	auto parse_res = json_walk_collector::parse_fully(
139	ldh_line_value.text_value_fragment());	×
140
141	if (parse_res.isOk()) {	×
142	this->ldh_json_pairs[ldh_line_value.lv_meta.lvm_name]	×
143	= parse_res.unwrap();	×
144	}
145	break;	×
146	}
147	case value_kind_t::VALUE_XML: {	1✔
148	auto col_name = ldh_line_value.lv_meta.lvm_name;	1✔
149	pugi::xml_document doc;	1✔
150
151	auto parse_res
152	= doc.load_buffer(ldh_line_value.text_value(),	1✔
153	ldh_line_value.text_length());
154
155	if (parse_res) {	1✔
156	pugi::xpath_query query("//*");	1✔
157	auto node_set = doc.select_nodes(query);	1✔
158
159	for (const auto& xpath_node : node_set) {	6✔
160	auto node_path = lnav::pugixml::get_actual_path(
161	xpath_node.node());	5✔
162	for (auto& attr : xpath_node.node().attributes()) {	6✔
163	auto attr_path
164	= fmt::format(FMT_STRING("{}/@{}"),	2✔
165	node_path,
166	attr.name());	1✔
167
168	this->ldh_xml_pairs[std::make_pair(col_name,	1✔
169	attr_path)]	2✔
170	= attr.value();	2✔
171	}	1✔
172
173	if (xpath_node.node().text().empty()) {	5✔
174	continue;	2✔
175	}
176
177	auto text_path = fmt::format(
178	FMT_STRING("{}/text()"), node_path);	12✔
179	this->ldh_xml_pairs[std::make_pair(col_name,	3✔
180	text_path)]	6✔
181	= trim(xpath_node.node().text().get());	9✔
182	}	5✔
183	}	1✔
184	break;	1✔
185	}	1✔
186	default:	154✔
187	break;	154✔
188	}
189	}
190
191	retval = true;	22✔
192	}	22✔
193
194	return retval;	22✔
195	}
196
197	void
198	log_data_helper::parse_body()	15✔
199	{
200	if (!this->ldh_line->is_message()) {	15✔
201	return;	×
202	}
203
204	if (!this->ldh_line_values.lvv_sbr.get_metadata().m_valid_utf) {	15✔
205	return;	×
206	}
207
208	auto& sbr = this->ldh_line_values.lvv_sbr;	15✔
209	auto& sa = this->ldh_line_attrs;	15✔
210	auto body = find_string_attr_range(sa, &SA_BODY);	15✔
211	if (body.lr_start == -1) {	15✔
212	body.lr_start = this->ldh_line_values.lvv_sbr.length();	×
213	body.lr_end = this->ldh_line_values.lvv_sbr.length();	×
214	}
215	auto body_sf = sbr.to_string_fragment(body);	15✔
216	#ifdef HAVE_RUST_DEPS
217	auto file_rust_str = rust::Str();	15✔
218	auto lineno = 0UL;	15✔
219	auto body_rust_str = rust::Str(body_sf.data(), body_sf.length());	15✔
220	auto src_file = find_string_attr_range(sa, &SA_SRC_FILE);	15✔
221	if (src_file.is_valid()) {	15✔
222	auto src_file_sf = sbr.to_string_fragment(src_file);	×
223	file_rust_str = rust::Str(src_file_sf.data(), src_file_sf.length());	×
224	}
225	auto src_line = find_string_attr_range(sa, &SA_SRC_LINE);	15✔
226	if (src_line.is_valid()) {	15✔
227	auto src_line_sf = sbr.to_string_fragment(src_line);	×
228	auto scan_res
229	= scn::scan_int<decltype(lineno)>(src_line_sf.to_string_view());	×
230	if (scan_res) {	×
231	lineno = scan_res->value();	×
232	}
233	}
234	this->ldh_src_ref = std::nullopt;	15✔
235	this->ldh_src_vars.clear();	15✔
236	auto find_res
237	= lnav_rs_ext::find_log_statement(file_rust_str, lineno, body_rust_str);	15✔
238	if (find_res != nullptr) {	15✔
239	this->ldh_src_ref = lnav::src_ref{	×
240	std::filesystem::path((std::string) find_res->src.file),	×
241	(uint32_t) find_res->src.begin_line,	×
242	(std::string) find_res->src.name,	×
243	};
244	for (const auto& [expr, value] : find_res->variables) {	×
245	this->ldh_src_vars.emplace_back((std::string) expr,	×
246	(std::string) value);	×
247	}
248	} else
249	#endif
250	{
251	this->ldh_scanner = std::make_unique<data_scanner>(body_sf);	15✔
252	this->ldh_parser
253	= std::make_unique<data_parser>(this->ldh_scanner.get());	15✔
254	this->ldh_msg_format.clear();	15✔
255	this->ldh_parser->dp_msg_format = &this->ldh_msg_format;	15✔
256	if (body.length() < 128 * 1024) {	15✔
257	this->ldh_parser->parse();	15✔
258	}
259	this->ldh_namer
260	= std::make_unique<column_namer>(column_namer::language::SQL);	15✔
261	for (const auto& lv : this->ldh_line_values.lvv_values) {	135✔
262	this->ldh_namer->cn_builtin_names.emplace_back(	120✔
263	lv.lv_meta.lvm_name.get());	120✔
264	}
265	}
266	}	15✔
267
268	int
269	log_data_helper::get_line_bounds(size_t& line_index_out,	×
270	size_t& line_end_index_out) const
271	{
272	int retval = 0;	×
273
274	line_end_index_out = 0;	×
275	do {
276	line_index_out = line_end_index_out;	×
277	const auto* line_end = (const char*) memchr(	×
278	this->ldh_line_values.lvv_sbr.get_data() + line_index_out + 1,	×
279	'\n',
280	this->ldh_line_values.lvv_sbr.length() - line_index_out - 1);	×
281	if (line_end != nullptr) {	×
282	line_end_index_out
283	= line_end - this->ldh_line_values.lvv_sbr.get_data();	×
284	} else {
285	line_end_index_out = std::string::npos;	×
286	}
287	retval += 1;	×
288	} while (retval <= this->ldh_y_offset);	×
289
290	if (line_end_index_out == std::string::npos) {	×
291	line_end_index_out = this->ldh_line_values.lvv_sbr.length();	×
292	}
293
294	return retval;	×
295	}
296
297	std::string
298	log_data_helper::format_json_getter(const intern_string_t field, int index)	×
299	{
300	std::string retval;	×
301
302	auto qname = sql_quote_ident(field.get());	×
303	retval = lnav::sql::mprintf(	×
304	"jget(%s,%Q)",
305	qname.in(),
306	this->ldh_json_pairs[field].jwc_values[index].first.c_str());	×
307
308	return retval;	×
309	}

tstack / lnav / 20284884426-2753

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous