• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tstack / lnav / 17589970077-2502

09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%
17589970077-2502

push

github

tstack
[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.55
/src/data_parser.hh
1
/**
2
 * Copyright (c) 2007-2012, Timothy Stack
3
 *
4
 * All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without
7
 * modification, are permitted provided that the following conditions are met:
8
 *
9
 * * Redistributions of source code must retain the above copyright notice, this
10
 * list of conditions and the following disclaimer.
11
 * * Redistributions in binary form must reproduce the above copyright notice,
12
 * this list of conditions and the following disclaimer in the documentation
13
 * and/or other materials provided with the distribution.
14
 * * Neither the name of Timothy Stack nor the names of its contributors
15
 * may be used to endorse or promote products derived from this software
16
 * without specific prior written permission.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
 */
29

30
#ifndef data_parser_hh
31
#define data_parser_hh
32

33
#include <iterator>
34
#include <list>
35
#include <vector>
36

37
#include <stdio.h>
38

39
#include "base/lnav_log.hh"
40
#include "byte_array.hh"
41
#include "data_scanner.hh"
42

43
#define ELEMENT_LIST_T(var) var("" #var, __FILE__, __LINE__, group_depth)
44
#define PUSH_FRONT(elem)    push_front(elem, __FILE__, __LINE__)
45
#define PUSH_BACK(elem)     push_back(elem, __FILE__, __LINE__)
46
#define POP_FRONT(elem)     pop_front(__FILE__, __LINE__)
47
#define POP_BACK(elem)      pop_back(__FILE__, __LINE__)
48
#define CLEAR(elem)         clear2(__FILE__, __LINE__)
49
#define SWAP(other)         swap(other, __FILE__, __LINE__)
50
#define SPLICE(pos, other, first, last) \
51
    splice(pos, other, first, last, __FILE__, __LINE__)
52

53
template<class Container, class UnaryPredicate>
54
void
55
strip(Container& container, UnaryPredicate p)
1,716✔
56
{
57
    while (!container.empty() && p(container.front())) {
1,716✔
UNCOV
58
        container.POP_FRONT();
×
59
    }
60
    while (!container.empty() && p(container.back())) {
1,716✔
UNCOV
61
        container.POP_BACK();
×
62
    }
63
}
1,716✔
64

65
enum data_format_state_t {
66
    DFS_ERROR = -1,
67
    DFS_INIT,
68
    DFS_KEY,
69
    DFS_EXPECTING_SEP,
70
    DFS_VALUE,
71
};
72

73
struct data_format {
74
    data_format(const char* name = nullptr,
18,282✔
75
                data_token_t appender = DT_INVALID,
76
                data_token_t terminator = DT_INVALID) noexcept
77
        : df_name(name), df_appender(appender), df_terminator(terminator),
18,282✔
78
          df_qualifier(DT_INVALID), df_separator(DT_COLON),
18,282✔
79
          df_prefix_terminator(DT_INVALID)
18,282✔
80
    {
81
    }
18,282✔
82

83
    const char* df_name;
84
    data_token_t df_appender;
85
    data_token_t df_terminator;
86
    data_token_t df_qualifier;
87
    data_token_t df_separator;
88
    data_token_t df_prefix_terminator;
89
};
90

91
data_format_state_t dfs_prefix_next(data_format_state_t state,
92
                                    data_token_t next_token);
93
data_format_state_t dfs_semi_next(data_format_state_t state,
94
                                  data_token_t next_token);
95
data_format_state_t dfs_comma_next(data_format_state_t state,
96
                                   data_token_t next_token);
97

98
#define LIST_INIT_TRACE \
99
    do { \
100
        if (TRACE_FILE != NULL) { \
101
            fprintf(TRACE_FILE, \
102
                    "%p %s:%d %s %s %d\n", \
103
                    this, \
104
                    fn, \
105
                    line, \
106
                    __func__, \
107
                    varname, \
108
                    group_depth); \
109
        } \
110
    } while (false)
111

112
#define LIST_DEINIT_TRACE \
113
    do { \
114
        if (TRACE_FILE != NULL) { \
115
            fprintf(TRACE_FILE, "%p %s:%d %s\n", this, fn, line, __func__); \
116
        } \
117
    } while (false)
118

119
#define ELEMENT_TRACE \
120
    do { \
121
        if (TRACE_FILE != NULL) { \
122
            fprintf(TRACE_FILE, \
123
                    "%p %s:%d %s %s %d:%d\n", \
124
                    this, \
125
                    fn, \
126
                    line, \
127
                    __func__, \
128
                    data_scanner::token2name(elem.e_token), \
129
                    elem.e_capture.c_begin, \
130
                    elem.e_capture.c_end); \
131
        } \
132
    } while (false)
133

134
#define LIST_TRACE \
135
    do { \
136
        if (TRACE_FILE != NULL) { \
137
            fprintf(TRACE_FILE, "%p %s:%d %s\n", this, fn, line, __func__); \
138
        } \
139
    } while (false)
140

141
#define SPLICE_TRACE \
142
    do { \
143
        if (TRACE_FILE != NULL) { \
144
            fprintf(TRACE_FILE, \
145
                    "%p %s:%d %s %d %p %d:%d\n", \
146
                    this, \
147
                    fn, \
148
                    line, \
149
                    __func__, \
150
                    (int) std::distance(this->begin(), pos), \
151
                    &other, \
152
                    (int) std::distance(other.begin(), first), \
153
                    (int) std::distance(last, other.end())); \
154
        } \
155
    } while (false);
156

157
#define SWAP_TRACE(other) \
158
    do { \
159
        if (TRACE_FILE != NULL) { \
160
            fprintf(TRACE_FILE, \
161
                    "%p %s:%d %s %p\n", \
162
                    this, \
163
                    fn, \
164
                    line, \
165
                    __func__, \
166
                    &other); \
167
        } \
168
    } while (false);
169

170
#define POINT_TRACE(name) \
171
    do { \
172
        if (TRACE_FILE) { \
173
            fprintf( \
174
                TRACE_FILE, "0x0 %s:%d point %s\n", __FILE__, __LINE__, name); \
175
        } \
176
    } while (false);
177

178
#define FORMAT_TRACE(elist) \
179
    do { \
180
        if (TRACE_FILE) { \
181
            const data_format& df = elist.el_format; \
182
            fprintf(TRACE_FILE, \
183
                    "%p %s:%d format %d %s %s %s %s %s\n", \
184
                    &elist, \
185
                    __FILE__, \
186
                    __LINE__, \
187
                    group_depth, \
188
                    data_scanner::token2name(df.df_appender), \
189
                    data_scanner::token2name(df.df_terminator), \
190
                    data_scanner::token2name(df.df_qualifier), \
191
                    data_scanner::token2name(df.df_separator), \
192
                    data_scanner::token2name(df.df_prefix_terminator)); \
193
        } \
194
    } while (false);
195

196
#define CONSUMED_TRACE(elist) \
197
    do { \
198
        if (TRACE_FILE) { \
199
            fprintf(TRACE_FILE, \
200
                    "%p %s:%d consumed\n", \
201
                    &elist, \
202
                    __FILE__, \
203
                    __LINE__); \
204
        } \
205
    } while (false);
206

207
class data_parser {
208
public:
209
    static data_format FORMAT_SEMI;
210
    static data_format FORMAT_COMMA;
211
    static data_format FORMAT_EMDASH;
212
    static data_format FORMAT_PLAIN;
213

214
    static FILE* TRACE_FILE;
215

216
    typedef byte_array<2, uint64_t> schema_id_t;
217

218
    struct element;
219
    /* typedef std::list<element> element_list_t; */
220

221
    class element_list_t : public std::list<element> {
222
    public:
223
        element_list_t(const char* varname,
12,181✔
224
                       const char* fn,
225
                       int line,
226
                       int group_depth = -1)
227
        {
12,181✔
228
            LIST_INIT_TRACE;
12,181✔
229
        }
12,181✔
230

231
        element_list_t()
1,013✔
232
        {
1,013✔
233
            const char* varname = "_anon2_";
1,013✔
234
            const char* fn = __FILE__;
1,013✔
235
            int line = __LINE__;
1,013✔
236
            int group_depth = -1;
1,013✔
237

238
            LIST_INIT_TRACE;
1,013✔
239
        }
1,013✔
240

241
        element_list_t(const element_list_t& other) : std::list<element>(other)
242
        {
243
            this->el_format = other.el_format;
244
        }
245

246
        ~element_list_t()
9,592✔
247
        {
248
            const char* fn = __FILE__;
9,592✔
249
            int line = __LINE__;
9,592✔
250

251
            LIST_DEINIT_TRACE;
9,592✔
252
        }
9,592✔
253

254
        void push_front(const element& elem, const char* fn, int line)
270✔
255
        {
256
            ELEMENT_TRACE;
270✔
257

258
            require(elem.e_capture.c_end >= -1);
270✔
259
            this->std::list<element>::push_front(elem);
270✔
260
        }
270✔
261

262
        void push_back(const element& elem, const char* fn, int line);
263

264
        void pop_front(const char* fn, int line)
1,584✔
265
        {
266
            LIST_TRACE;
1,584✔
267

268
            this->std::list<element>::pop_front();
1,584✔
269
        }
1,584✔
270

271
        void pop_back(const char* fn, int line)
211✔
272
        {
273
            LIST_TRACE;
211✔
274

275
            this->std::list<element>::pop_back();
211✔
276
        }
211✔
277

278
        void clear2(const char* fn, int line)
1,792✔
279
        {
280
            LIST_TRACE;
1,792✔
281

282
            this->std::list<element>::clear();
1,792✔
283
        }
1,792✔
284

285
        void swap(element_list_t& other, const char* fn, int line)
4,313✔
286
        {
287
            SWAP_TRACE(other);
4,313✔
288

289
            this->std::list<element>::swap(other);
4,313✔
290
        }
4,313✔
291

292
        void splice(iterator pos,
1,799✔
293
                    element_list_t& other,
294
                    iterator first,
295
                    iterator last,
296
                    const char* fn,
297
                    int line)
298
        {
299
            SPLICE_TRACE;
3,683✔
300

301
            this->std::list<element>::splice(pos, other, first, last);
1,799✔
302
        }
1,799✔
303

304
        data_format el_format;
305
    };
306

307
    struct element {
308
        element();
309

310
        element(element_list_t& subs,
311
                data_token_t token,
312
                bool assign_subs_elements = true);
313

314
        element(const element& other);
315

316
        ~element();
317

318
        element& operator=(const element& other);
319

320
        void assign_elements(element_list_t& subs);
321

322
        void update_capture();
323

324
        const element& get_pair_value() const;
325

326
        data_token_t value_token() const;
327

328
        const element& get_value_elem() const;
329

330
        const element& get_pair_elem() const;
331

332
        bool is_value() const;
333

334
        void print(FILE* out, data_scanner&, int offset = 0) const;
335

336
        data_scanner::capture_t e_capture;
337
        data_token_t e_token;
338

339
        element_list_t* e_sub_elements;
340
    };
341

342
    struct element_cmp {
343
        bool operator()(data_token_t token, const element& elem) const
344
        {
345
            return token == elem.e_token || token == DT_ANY;
346
        }
347

348
        bool operator()(const element& elem, data_token_t token) const
349
        {
350
            return (*this)(token, elem);
351
        }
352
    };
353

354
    struct element_if {
355
        element_if(data_token_t token) : ei_token(token) {}
788✔
356

357
        bool operator()(const element& a) const
277✔
358
        {
359
            return a.e_token == this->ei_token;
277✔
360
        }
361

362
    private:
363
        data_token_t ei_token;
364
    };
365

366
    struct element_is_space {
367
        bool operator()(const element& el) const
1,234✔
368
        {
369
            return el.e_token == DT_WHITE || el.e_token == DT_CSI;
1,234✔
370
        }
371
    };
372

373
    struct discover_format_state {
374
        discover_format_state();
375

376
        void update_for_element(const element& elem);
377

378
        void finalize();
379

380
        data_format_state_t dfs_prefix_state;
381
        data_format_state_t dfs_semi_state;
382
        data_format_state_t dfs_comma_state;
383
        int dfs_hist[DT_TERMINAL_MAX];
384

385
        data_format dfs_format;
386
    };
387

388
    data_parser(data_scanner* ds);
389

390
    void pairup(schema_id_t* schema,
391
                element_list_t& pairs_out,
392
                element_list_t& in_list,
393
                int group_depth = 0);
394

395
    void discover_format();
396

397
    void end_of_value(element_list_t& el_stack,
398
                      element_list_t& key_comps,
399
                      element_list_t& value,
400
                      const element_list_t& in_list,
401
                      int group_depth,
402
                      element_list_t::iterator iter);
403

404
    void parse();
405

406
    std::string get_element_string(const element& elem) const;
407

408
    std::string get_string_up_to_value(const element& elem);
409

410
    const char* get_element_string(const element& elem, size_t& len_out);
411

412
    void print(FILE* out, element_list_t& el);
413

414
    std::vector<data_token_t> dp_group_token;
415
    std::list<element_list_t> dp_group_stack;
416

417
    element_list_t dp_errors;
418

419
    element_list_t dp_pairs;
420
    schema_id_t dp_schema_id;
421
    std::string* dp_msg_format;
422
    int dp_msg_format_begin;
423

424
private:
425
    data_scanner* dp_scanner;
426
};
427

428
#endif
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc