17589970077-2502

Committed 09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%

Build # 17589970077-2502

Build Type

push

github

Committed by

tstack

Commit Message

[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

Run Details

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.14

/src/file_format.cc

/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file file_format.hh
 */

#include "file_format.hh"

#include "archive_manager.hh"
#include "base/auto_fd.hh"
#include "base/fs_util.hh"
#include "base/intern_string.hh"
#include "base/lnav_log.hh"
#include "config.h"
#include "line_buffer.hh"
#include "piper.match.hh"
#include "text_format.hh"

detect_file_format_result
detect_file_format(const std::filesystem::path& filename)
{
    log_debug("detecting format of file: %s", filename.c_str());

    detect_file_format_result retval = {file_format_t::UNKNOWN};
    auto describe_res = archive_manager::describe(filename);
    if (describe_res.isOk()
        && describe_res.unwrap().is<archive_manager::archive_info>())
    {
        return {file_format_t::ARCHIVE};
    }

    auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);
    if (open_res.isErr()) {
        log_error("unable to open file for format detection: %s -- %s",
                  filename.c_str(),
                  open_res.unwrapErr().c_str());
    } else {
        auto fd = open_res.unwrap();
        uint8_t buffer[32];
        auto rc = read(fd, buffer, sizeof(buffer));

        if (rc < 0) {
            log_error("unable to read file for format detection: %s -- %s",
                      filename.c_str(),
                      strerror(errno));
        } else {
            static auto SQLITE3_HEADER = "SQLite format 3";
            auto header_frag = string_fragment::from_bytes(buffer, rc);

            if (header_frag.startswith(SQLITE3_HEADER)) {
                log_info("%s: appears to be a SQLite DB", filename.c_str());
                retval.dffr_file_format = file_format_t::SQLITE_DB;
            } else {
                auto tf = detect_text_format(header_frag, filename);
                auto looping = true;

                switch (tf) {
                    case text_format_t::TF_UNKNOWN:
                    case text_format_t::TF_BINARY:
                    case text_format_t::TF_LOG:
                    case text_format_t::TF_JSON:
                        log_info("file does not have a known text format: %s",
                                 filename.c_str());
                        break;
                    default:
                        log_info("file has text format: %s -> %d",
                                 filename.c_str(),
                                 tf);
                        looping = false;
                        break;
                }

                lnav::piper::multiplex_matcher mm;
                file_range next_range;
                line_buffer lb;
                lb.set_fd(fd);

                while (looping) {
                    auto load_res = lb.load_next_line(next_range);
                    if (load_res.isErr()) {
                        log_error(
                            "unable to load line for demux matching: %s -- %s",
                            filename.c_str(),
                            load_res.unwrapErr().c_str());
                        break;
                    }
                    if (!lb.is_header_utf8()) {
                        log_info("file is not UTF-8: %s", filename.c_str());
                        break;
                    }
                    if (lb.is_piper()) {
                        log_info("skipping demux match for piper file: %s",
                                 filename.c_str());
                        break;
                    }
                    const auto li = load_res.unwrap();
                    if (li.li_partial) {
                        log_info("skipping demux match for partial line");
                        break;
                    }
                    auto read_res = lb.read_range(li.li_file_range);
                    if (read_res.isErr()) {
                        log_error(
                            "unable to read line for demux matching: %s -- %s",
                            filename.c_str(),
                            read_res.unwrapErr().c_str());
                        break;
                    }
                    auto sbr = read_res.unwrap();
                    auto match_res = mm.match(sbr.to_string_fragment());

                    looping = match_res.match(
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_regex f) {
                            log_info("%s: is multiplexed using pattern %s",
                                     filename.c_str(),
                                     f.f_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_json f) {
                            log_info("%s: is multiplexed using JSON %s",
                                     filename.c_str(),
                                     f.fj_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::not_found nf) {
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::partial p) {
                            return true;
                        });

                    next_range = li.li_file_range;
                }
                retval.dffr_details = std::move(mm.mm_details);
            }
        }
    }

    return retval;
}

1	/**
2	* Copyright (c) 2020, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file file_format.hh
30	*/
31
32	#include "file_format.hh"
33
34	#include "archive_manager.hh"
35	#include "base/auto_fd.hh"
36	#include "base/fs_util.hh"
37	#include "base/intern_string.hh"
38	#include "base/lnav_log.hh"
39	#include "config.h"
40	#include "line_buffer.hh"
41	#include "piper.match.hh"
42	#include "text_format.hh"
43
44	detect_file_format_result
45	detect_file_format(const std::filesystem::path& filename)	575✔
46	{
47	log_debug("detecting format of file: %s", filename.c_str());	575✔
48
49	detect_file_format_result retval = {file_format_t::UNKNOWN};	575✔
50	auto describe_res = archive_manager::describe(filename);	575✔
51	if (describe_res.isOk()	1,150✔
52	&& describe_res.unwrap().is<archive_manager::archive_info>())	575✔
53	{
UNCOV 54	return {file_format_t::ARCHIVE};	×
55	}
56
57	auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);	575✔
58	if (open_res.isErr()) {	575✔
UNCOV 59	log_error("unable to open file for format detection: %s -- %s",	×
60	filename.c_str(),
61	open_res.unwrapErr().c_str());
62	} else {
63	auto fd = open_res.unwrap();	575✔
64	uint8_t buffer[32];
65	auto rc = read(fd, buffer, sizeof(buffer));	575✔
66
67	if (rc < 0) {	575✔
UNCOV 68	log_error("unable to read file for format detection: %s -- %s",	×
69	filename.c_str(),
70	strerror(errno));
71	} else {
72	static auto SQLITE3_HEADER = "SQLite format 3";
73	auto header_frag = string_fragment::from_bytes(buffer, rc);	575✔
74
75	if (header_frag.startswith(SQLITE3_HEADER)) {	575✔
76	log_info("%s: appears to be a SQLite DB", filename.c_str());	1✔
77	retval.dffr_file_format = file_format_t::SQLITE_DB;	1✔
78	} else {
79	auto tf = detect_text_format(header_frag, filename);	574✔
80	auto looping = true;	574✔
81
82	switch (tf) {	574✔
83	case text_format_t::TF_UNKNOWN:	553✔
84	case text_format_t::TF_BINARY:
85	case text_format_t::TF_LOG:
86	case text_format_t::TF_JSON:
87	log_info("file does not have a known text format: %s",	553✔
88	filename.c_str());
89	break;	553✔
90	default:	21✔
91	log_info("file has text format: %s -> %d",	21✔
92	filename.c_str(),
93	tf);
94	looping = false;	21✔
95	break;	21✔
96	}
97
98	lnav::piper::multiplex_matcher mm;	1,148✔
99	file_range next_range;	574✔
100	line_buffer lb;	1,148✔
101	lb.set_fd(fd);	574✔
102
103	while (looping) {	1,051✔
104	auto load_res = lb.load_next_line(next_range);	553✔
105	if (load_res.isErr()) {	553✔
106	log_error(	×
107	"unable to load line for demux matching: %s -- %s",
108	filename.c_str(),
109	load_res.unwrapErr().c_str());
UNCOV 110	break;	×
111	}
112	if (!lb.is_header_utf8()) {	553✔
113	log_info("file is not UTF-8: %s", filename.c_str());	4✔
114	break;	4✔
115	}
116	if (lb.is_piper()) {	549✔
117	log_info("skipping demux match for piper file: %s",	58✔
118	filename.c_str());
119	break;	58✔
120	}
121	const auto li = load_res.unwrap();	491✔
122	if (li.li_partial) {	491✔
123	log_info("skipping demux match for partial line");	14✔
124	break;	14✔
125	}
126	auto read_res = lb.read_range(li.li_file_range);	477✔
127	if (read_res.isErr()) {	477✔
128	log_error(	×
129	"unable to read line for demux matching: %s -- %s",
130	filename.c_str(),
131	read_res.unwrapErr().c_str());
UNCOV 132	break;	×
133	}
134	auto sbr = read_res.unwrap();	477✔
135	auto match_res = mm.match(sbr.to_string_fragment());	477✔
136
137	looping = match_res.match(	954✔
UNCOV 138	[&retval, &filename](	×
139	lnav::piper::multiplex_matcher::found_regex f) {
140	log_info("%s: is multiplexed using pattern %s",	5✔
141	filename.c_str(),
142	f.f_id.c_str());
143	retval.dffr_file_format	5✔
144	= file_format_t::MULTIPLEXED;	5✔
145	return false;	5✔
146	},
147	[&retval, &filename](	477✔
148	lnav::piper::multiplex_matcher::found_json f) {
149	log_info("%s: is multiplexed using JSON %s",	3✔
150	filename.c_str(),
151	f.fj_id.c_str());
152	retval.dffr_file_format	3✔
153	= file_format_t::MULTIPLEXED;	3✔
154	return false;	3✔
155	},
UNCOV 156	[](lnav::piper::multiplex_matcher::not_found nf) {	×
157	return false;	469✔
158	},
UNCOV 159	[](lnav::piper::multiplex_matcher::partial p) {	×
UNCOV 160	return true;	×
161	});
162
163	next_range = li.li_file_range;	477✔
164	}	553✔
165	retval.dffr_details = std::move(mm.mm_details);	574✔
166	}
167	}
168	}	575✔
169
170	return retval;	575✔
171	}	575✔

tstack / lnav / 17589970077-2502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous