17589970077-2502

Committed 09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%

Build # 17589970077-2502

Build Type

push

github

Committed by

tstack

Commit Message

[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

Run Details

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.81

/src/data_scanner.hh

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef data_scanner_hh
#define data_scanner_hh

#include <string>

#include "pcrepp/pcre2pp.hh"
#include "shared_buffer.hh"
#include "text_format.hh"

enum data_token_t {
    DT_INVALID = -1,

    DT_QUOTED_STRING = 0,
    DT_COMMENT,
    DT_URL,
    DT_PATH,
    DT_MAC_ADDRESS,
    DT_DATE,
    DT_TIME,
    DT_DATE_TIME,
    DT_IPV6_ADDRESS,
    DT_HEX_DUMP,
    DT_XML_DECL_TAG,
    DT_XML_EMPTY_TAG,
    DT_XML_OPEN_TAG,
    DT_XML_CLOSE_TAG,

    DT_H1,
    DT_H2,
    DT_H3,

    /* DT_QUALIFIED_NAME, */

    DT_COLON,
    DT_EQUALS,
    DT_COMMA,
    DT_SEMI,
    DT_EMDASH,

    DT_EMPTY_CONTAINER,

    DT_LCURLY,
    DT_RCURLY,

    DT_LSQUARE,
    DT_RSQUARE,

    DT_LPAREN,
    DT_RPAREN,

    DT_LANGLE,
    DT_RANGLE,

    DT_IPV4_ADDRESS,
    DT_UUID,

    DT_CREDIT_CARD_NUMBER,
    DT_VERSION_NUMBER,
    DT_OCTAL_NUMBER,
    DT_PERCENTAGE,
    DT_NUMBER,
    DT_HEX_NUMBER,

    DT_EMAIL,
    DT_CONSTANT,
    DT_WORD,
    DT_ID,
    DT_SYMBOL,
    DT_UNIT,
    DT_LINE,
    DT_WHITE,
    DT_DOT,
    DT_ESCAPED_CHAR,
    DT_CSI,

    DT_GARBAGE,
    DT_ZERO_WIDTH_SPACE,

    DT_DIFF_FILE_HEADER,
    DT_DIFF_HUNK_HEADING,

    DT_CODE_BLOCK,

    DT_ANCHOR,

    DT_TERMINAL_MAX = DT_ANCHOR + 1,

    DNT_KEY = 56,
    DNT_PAIR,
    DNT_VALUE,
    DNT_ROW,
    DNT_UNITS,
    DNT_MEASUREMENT,
    DNT_VARIABLE_KEY,
    DNT_ROWRANGE,
    DNT_GROUP,

    DNT_MAX,

    DT_ANY = 100,
};

class data_scanner {
public:
    static const char* token2name(data_token_t token);

    struct capture_t {
        capture_t() { /* We don't initialize anything since it's a perf hit. */
        }

        capture_t(int begin, int end) : c_begin(begin), c_end(end)
        {
            assert(begin <= end);
        }

        int c_begin;
        int c_end;

        void ltrim(const char* str);

        bool contains(int pos) const
        {
            return this->c_begin <= pos && pos < this->c_end;
        }

        bool is_valid() const { return this->c_begin != -1; }

        int length() const { return this->c_end - this->c_begin; }

        bool empty() const { return this->c_begin == this->c_end; }
    };

    data_scanner(const std::string& line, size_t off = 0)
        : ds_line(line), ds_input(this->ds_line), ds_init_offset(off),
          ds_next_offset(off)
    {
        this->cleanup_end();
    }

    explicit data_scanner(string_fragment sf, bool do_cleanup = true)
        : ds_input(sf), ds_init_offset(sf.sf_begin), ds_next_offset(sf.sf_begin)
    {
        if (do_cleanup) {
            this->cleanup_end();
        }
    }

    explicit data_scanner(const shared_buffer_ref& line, size_t off, size_t end)
        : ds_sbr(line.clone()),
          ds_input(line.to_string_fragment().sub_range(0, end)),
          ds_init_offset(off), ds_next_offset(off)
    {
        this->cleanup_end();
    }

    struct tokenize_result {
        data_token_t tr_token{DT_INVALID};
        capture_t tr_capture;
        capture_t tr_inner_capture;
        const char* tr_data{nullptr};

        string_fragment to_string_fragment() const
        {
            return string_fragment::from_byte_range(this->tr_data,
                                                    this->tr_capture.c_begin,
                                                    this->tr_capture.c_end);
        }

        string_fragment inner_string_fragment() const
        {
            return string_fragment::from_byte_range(
                this->tr_data,
                this->tr_inner_capture.c_begin,
                this->tr_inner_capture.c_end);
        }

        std::string to_string() const
        {
            return {
                &this->tr_data[this->tr_capture.c_begin],
                (size_t) this->tr_capture.length(),
            };
        }
    };

    std::optional<tokenize_result> tokenize2(text_format_t tf
                                             = text_format_t::TF_UNKNOWN);

    std::optional<tokenize_result> find_matching_bracket(text_format_t tf,
                                                         tokenize_result tr);

    void reset() { this->ds_next_offset = this->ds_init_offset; }

    int get_init_offset() const { return this->ds_init_offset; }

    string_fragment get_input() const { return this->ds_input; }

    string_fragment to_string_fragment(capture_t cap) const
    {
        return string_fragment{
            this->ds_input.sf_string, cap.c_begin, cap.c_end};
    }

private:
    void cleanup_end();

    bool is_credit_card(string_fragment frag) const;

    std::optional<tokenize_result> tokenize_int(text_format_t tf
                                                = text_format_t::TF_UNKNOWN);

    std::string ds_line;
    shared_buffer_ref ds_sbr;
    string_fragment ds_input;
    int ds_init_offset{0};
    int ds_next_offset{0};
    bool ds_bol{true};
    bool ds_units{false};
    std::vector<tokenize_result> ds_matching_brackets;
    bool ds_last_bracket_matched{false};
};

inline data_token_t
to_opener(data_token_t dt)
{
    switch (dt) {
        case DT_XML_CLOSE_TAG:
            return DT_XML_OPEN_TAG;
        case DT_RCURLY:
            return DT_LCURLY;
        case DT_RSQUARE:
            return DT_LSQUARE;
        case DT_RPAREN:
            return DT_LPAREN;
        default:
            ensure(0);
    }
}

inline data_token_t
to_closer(data_token_t dt)
{
    switch (dt) {
        case DT_XML_OPEN_TAG:
            return DT_XML_CLOSE_TAG;
        case DT_LCURLY:
            return DT_RCURLY;
        case DT_LSQUARE:
            return DT_RSQUARE;
        case DT_LPAREN:
            return DT_RPAREN;
        default:
            ensure(0);
    }
}

#endif

1	/**
2	* Copyright (c) 2007-2012, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*/
29
30	#ifndef data_scanner_hh
31	#define data_scanner_hh
32
33	#include <string>
34
35	#include "pcrepp/pcre2pp.hh"
36	#include "shared_buffer.hh"
37	#include "text_format.hh"
38
39	enum data_token_t {
40	DT_INVALID = -1,
41
42	DT_QUOTED_STRING = 0,
43	DT_COMMENT,
44	DT_URL,
45	DT_PATH,
46	DT_MAC_ADDRESS,
47	DT_DATE,
48	DT_TIME,
49	DT_DATE_TIME,
50	DT_IPV6_ADDRESS,
51	DT_HEX_DUMP,
52	DT_XML_DECL_TAG,
53	DT_XML_EMPTY_TAG,
54	DT_XML_OPEN_TAG,
55	DT_XML_CLOSE_TAG,
56
57	DT_H1,
58	DT_H2,
59	DT_H3,
60
61	/* DT_QUALIFIED_NAME, */
62
63	DT_COLON,
64	DT_EQUALS,
65	DT_COMMA,
66	DT_SEMI,
67	DT_EMDASH,
68
69	DT_EMPTY_CONTAINER,
70
71	DT_LCURLY,
72	DT_RCURLY,
73
74	DT_LSQUARE,
75	DT_RSQUARE,
76
77	DT_LPAREN,
78	DT_RPAREN,
79
80	DT_LANGLE,
81	DT_RANGLE,
82
83	DT_IPV4_ADDRESS,
84	DT_UUID,
85
86	DT_CREDIT_CARD_NUMBER,
87	DT_VERSION_NUMBER,
88	DT_OCTAL_NUMBER,
89	DT_PERCENTAGE,
90	DT_NUMBER,
91	DT_HEX_NUMBER,
92
93	DT_EMAIL,
94	DT_CONSTANT,
95	DT_WORD,
96	DT_ID,
97	DT_SYMBOL,
98	DT_UNIT,
99	DT_LINE,
100	DT_WHITE,
101	DT_DOT,
102	DT_ESCAPED_CHAR,
103	DT_CSI,
104
105	DT_GARBAGE,
106	DT_ZERO_WIDTH_SPACE,
107
108	DT_DIFF_FILE_HEADER,
109	DT_DIFF_HUNK_HEADING,
110
111	DT_CODE_BLOCK,
112
113	DT_ANCHOR,
114
115	DT_TERMINAL_MAX = DT_ANCHOR + 1,
116
117	DNT_KEY = 56,
118	DNT_PAIR,
119	DNT_VALUE,
120	DNT_ROW,
121	DNT_UNITS,
122	DNT_MEASUREMENT,
123	DNT_VARIABLE_KEY,
124	DNT_ROWRANGE,
125	DNT_GROUP,
126
127	DNT_MAX,
128
129	DT_ANY = 100,
130	};
131
132	class data_scanner {
133	public:
134	static const char* token2name(data_token_t token);
135
136	struct capture_t {
137	capture_t() { /* We don't initialize anything since it's a perf hit. */	163,379✔
138	}	163,379✔
139
140	capture_t(int begin, int end) : c_begin(begin), c_end(end)	7,996✔
141	{
142	assert(begin <= end);
143	}	7,996✔
144
145	int c_begin;
146	int c_end;
147
148	void ltrim(const char* str);
149
150	bool contains(int pos) const
151	{
152	return this->c_begin <= pos && pos < this->c_end;
153	}
154
155	bool is_valid() const { return this->c_begin != -1; }
156
157	int length() const { return this->c_end - this->c_begin; }	4,594✔
158
159	bool empty() const { return this->c_begin == this->c_end; }
160	};
161
162	data_scanner(const std::string& line, size_t off = 0)	75✔
163	: ds_line(line), ds_input(this->ds_line), ds_init_offset(off),	150✔
164	ds_next_offset(off)	150✔
165	{
166	this->cleanup_end();	75✔
167	}	75✔
168
169	explicit data_scanner(string_fragment sf, bool do_cleanup = true)	752✔
170	: ds_input(sf), ds_init_offset(sf.sf_begin), ds_next_offset(sf.sf_begin)	752✔
171	{
172	if (do_cleanup) {	752✔
173	this->cleanup_end();	752✔
174	}
175	}	752✔
176
177	explicit data_scanner(const shared_buffer_ref& line, size_t off, size_t end)	429✔
178	: ds_sbr(line.clone()),	429✔
179	ds_input(line.to_string_fragment().sub_range(0, end)),	429✔
180	ds_init_offset(off), ds_next_offset(off)	858✔
181	{
182	this->cleanup_end();	429✔
183	}	429✔
184
185	struct tokenize_result {
186	data_token_t tr_token{DT_INVALID};
187	capture_t tr_capture;
188	capture_t tr_inner_capture;
189	const char* tr_data{nullptr};
190
191	string_fragment to_string_fragment() const	3,964✔
192	{
193	return string_fragment::from_byte_range(this->tr_data,	3,964✔
194	this->tr_capture.c_begin,	3,964✔
195	this->tr_capture.c_end);	3,964✔
196	}
197
UNCOV 198	string_fragment inner_string_fragment() const	×
199	{
UNCOV 200	return string_fragment::from_byte_range(	×
UNCOV 201	this->tr_data,	×
UNCOV 202	this->tr_inner_capture.c_begin,	×
UNCOV 203	this->tr_inner_capture.c_end);	×
204	}
205
206	std::string to_string() const	601✔
207	{
208	return {
209	&this->tr_data[this->tr_capture.c_begin],	601✔
210	(size_t) this->tr_capture.length(),	601✔
211	};	1,803✔
212	}
213	};
214
215	std::optional<tokenize_result> tokenize2(text_format_t tf
216	= text_format_t::TF_UNKNOWN);
217
218	std::optional<tokenize_result> find_matching_bracket(text_format_t tf,
219	tokenize_result tr);
220
221	void reset() { this->ds_next_offset = this->ds_init_offset; }	458✔
222
223	int get_init_offset() const { return this->ds_init_offset; }	876✔
224
225	string_fragment get_input() const { return this->ds_input; }	103✔
226
227	string_fragment to_string_fragment(capture_t cap) const	8,674✔
228	{
229	return string_fragment{	17,348✔
230	this->ds_input.sf_string, cap.c_begin, cap.c_end};	8,674✔
231	}
232
233	private:
234	void cleanup_end();
235
236	bool is_credit_card(string_fragment frag) const;
237
238	std::optional<tokenize_result> tokenize_int(text_format_t tf
239	= text_format_t::TF_UNKNOWN);
240
241	std::string ds_line;
242	shared_buffer_ref ds_sbr;
243	string_fragment ds_input;
244	int ds_init_offset{0};
245	int ds_next_offset{0};
246	bool ds_bol{true};
247	bool ds_units{false};
248	std::vector<tokenize_result> ds_matching_brackets;
249	bool ds_last_bracket_matched{false};
250	};
251
252	inline data_token_t
253	to_opener(data_token_t dt)	1,710✔
254	{
255	switch (dt) {	1,710✔
UNCOV 256	case DT_XML_CLOSE_TAG:	×
UNCOV 257	return DT_XML_OPEN_TAG;	×
258	case DT_RCURLY:	309✔
259	return DT_LCURLY;	309✔
260	case DT_RSQUARE:	567✔
261	return DT_LSQUARE;	567✔
262	case DT_RPAREN:	834✔
263	return DT_LPAREN;	834✔
UNCOV 264	default:	×
UNCOV 265	ensure(0);	×
266	}
267	}
268
269	inline data_token_t
270	to_closer(data_token_t dt)	946✔
271	{
272	switch (dt) {	946✔
273	case DT_XML_OPEN_TAG:	487✔
274	return DT_XML_CLOSE_TAG;	487✔
275	case DT_LCURLY:	214✔
276	return DT_RCURLY;	214✔
277	case DT_LSQUARE:	139✔
278	return DT_RSQUARE;	139✔
279	case DT_LPAREN:	106✔
280	return DT_RPAREN;	106✔
UNCOV 281	default:	×
UNCOV 282	ensure(0);	×
283	}
284	}
285
286	#endif

tstack / lnav / 17589970077-2502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous