19553931481-2697

Committed 20 Nov 2025 10:54PM UTC coverage: 68.864%. Remained the same

Build # 19553931481-2697

Build Type

push

github

Committed by

tstack

Commit Message

[files] some more files panel polishing

Run Details

8 of 17 new or added lines in 2 files covered. (47.06%)

1 existing line in 1 file now uncovered.

51097 of 74200 relevant lines covered (68.86%)

431705.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.53

/src/file_format.cc

/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file file_format.hh
 */

#include "file_format.hh"

#include "archive_manager.hh"
#include "base/auto_fd.hh"
#include "base/fs_util.hh"
#include "base/intern_string.hh"
#include "base/lnav.console.hh"
#include "base/lnav_log.hh"
#include "config.h"
#include "line_buffer.hh"
#include "piper.match.hh"
#include "text_format.hh"

detect_file_format_result
detect_file_format(const std::filesystem::path& filename)
{
    static const auto JAR_EXT = std::filesystem::path(".jar");

    // static auto op = lnav_operation{"detect_file_format"};
    // auto op_guard = lnav_opid_guard::internal(op);
    log_trace("detecting format of file: %s", filename.c_str());

    detect_file_format_result retval = {file_format_t::UNKNOWN};
    auto ext = filename.extension();

    if (ext == JAR_EXT) {
        static const auto JAR_MSG
            = lnav::console::user_message::info("ignoring Java JAR file");
        return {file_format_t::UNSUPPORTED, {JAR_MSG}};
    }

    auto describe_res = archive_manager::describe(filename);
    if (describe_res.isOk()) {
        auto describe_inner = describe_res.unwrap();
        if (describe_inner.is<archive_manager::archive_info>()) {
            auto& ai = describe_inner.get<archive_manager::archive_info>();
            auto um = lnav::console::user_message::info(
                attr_line_t()
                    .append_quoted(ai.ai_format_name)
                    .append(" archive with ")
                    .append(lnav::roles::number(
                        fmt::to_string(ai.ai_entries.size())))
                    .append(ai.ai_entries.size() == 1 ? " entry" : " entries"));
            return {file_format_t::ARCHIVE, {um}};
        }
    }

    auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);
    if (open_res.isErr()) {
        log_error("unable to open file for format detection: %s -- %s",
                  filename.c_str(),
                  open_res.unwrapErr().c_str());
    } else {
        auto fd = open_res.unwrap();
        uint8_t buffer[32];
        auto rc = read(fd, buffer, sizeof(buffer));

        if (rc < 0) {
            log_error("unable to read file for format detection: %s -- %s",
                      filename.c_str(),
                      strerror(errno));
        } else {
            static const auto* SQLITE3_HEADER = "SQLite format 3";
            static const auto* JAVA_CLASS_HEADER = "\xca\xfe\xba\xbe";

            auto header_frag = string_fragment::from_bytes(buffer, rc);

            if (header_frag.startswith(SQLITE3_HEADER)) {
                static const auto DB_MSG
                    = lnav::console::user_message::info("SQLite database file");

                log_info("%s: appears to be a SQLite DB", filename.c_str());
                retval.dffr_file_format = file_format_t::SQLITE_DB;
                retval.dffr_details.emplace_back(DB_MSG);
            } else if (header_frag.startswith(JAVA_CLASS_HEADER)) {
                static const auto CLASS_MSG = lnav::console::user_message::info(
                    "ignoring Java Class file");

                retval.dffr_file_format = file_format_t::UNSUPPORTED;
                retval.dffr_details.emplace_back(CLASS_MSG);
            } else {
                auto tf = detect_text_format(header_frag, filename);
                auto looping = true;

                switch (tf) {
                    case text_format_t::TF_UNKNOWN:
                    case text_format_t::TF_BINARY:
                    case text_format_t::TF_LOG:
                    case text_format_t::TF_JSON:
                        log_info("file does not have a known text format: %s",
                                 filename.c_str());
                        break;
                    default:
                        log_info("file has text format: %s -> %d",
                                 filename.c_str(),
                                 tf);
                        looping = false;
                        break;
                }

                lnav::piper::multiplex_matcher mm;
                file_range next_range;
                line_buffer lb;
                lb.set_fd(fd);

                while (looping) {
                    auto load_res = lb.load_next_line(next_range);
                    if (load_res.isErr()) {
                        log_error(
                            "unable to load line for demux matching: %s -- %s",
                            filename.c_str(),
                            load_res.unwrapErr().c_str());
                        break;
                    }
                    if (!lb.is_header_utf8()) {
                        log_info("file is not UTF-8: %s", filename.c_str());
                        break;
                    }
                    if (lb.is_piper()) {
                        log_info("skipping demux match for piper file: %s",
                                 filename.c_str());
                        break;
                    }
                    const auto li = load_res.unwrap();
                    if (li.li_partial) {
                        log_info("skipping demux match for partial line");
                        break;
                    }
                    auto read_res = lb.read_range(li.li_file_range);
                    if (read_res.isErr()) {
                        log_error(
                            "unable to read line for demux matching: %s -- %s",
                            filename.c_str(),
                            read_res.unwrapErr().c_str());
                        break;
                    }
                    auto sbr = read_res.unwrap();
                    auto match_res = mm.match(sbr.to_string_fragment());

                    looping = match_res.match(
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_regex f) {
                            log_info("%s: is multiplexed using pattern %s",
                                     filename.c_str(),
                                     f.f_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_json f) {
                            log_info("%s: is multiplexed using JSON %s",
                                     filename.c_str(),
                                     f.fj_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::not_found nf) {
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::partial p) {
                            return true;
                        });

                    next_range = li.li_file_range;
                }
                retval.dffr_details = std::move(mm.mm_details);
            }
        }
    }

    return retval;
}

1	/**
2	* Copyright (c) 2020, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file file_format.hh
30	*/
31
32	#include "file_format.hh"
33
34	#include "archive_manager.hh"
35	#include "base/auto_fd.hh"
36	#include "base/fs_util.hh"
37	#include "base/intern_string.hh"
38	#include "base/lnav.console.hh"
39	#include "base/lnav_log.hh"
40	#include "config.h"
41	#include "line_buffer.hh"
42	#include "piper.match.hh"
43	#include "text_format.hh"
44
45	detect_file_format_result
46	detect_file_format(const std::filesystem::path& filename)	619✔
47	{
48	static const auto JAR_EXT = std::filesystem::path(".jar");	619✔
49
50	// static auto op = lnav_operation{"detect_file_format"};
51	// auto op_guard = lnav_opid_guard::internal(op);
52	log_trace("detecting format of file: %s", filename.c_str());	619✔
53
54	detect_file_format_result retval = {file_format_t::UNKNOWN};	619✔
55	auto ext = filename.extension();	619✔
56
57	if (ext == JAR_EXT) {	619✔
58	static const auto JAR_MSG
59	= lnav::console::user_message::info("ignoring Java JAR file");	×
60	return {file_format_t::UNSUPPORTED, {JAR_MSG}};	×
61	}
62
63	auto describe_res = archive_manager::describe(filename);	619✔
64	if (describe_res.isOk()) {	619✔
65	auto describe_inner = describe_res.unwrap();	619✔
66	if (describe_inner.is<archive_manager::archive_info>()) {	619✔
NEW 67	auto& ai = describe_inner.get<archive_manager::archive_info>();	×
68	auto um = lnav::console::user_message::info(
NEW 69	attr_line_t()	×
NEW 70	.append_quoted(ai.ai_format_name)	×
NEW 71	.append(" archive with ")	×
NEW 72	.append(lnav::roles::number(	×
NEW 73	fmt::to_string(ai.ai_entries.size())))	×
NEW 74	.append(ai.ai_entries.size() == 1 ? " entry" : " entries"));	×
NEW 75	return {file_format_t::ARCHIVE, {um}};	×
76	}
77	}	619✔
78
79	auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);	619✔
80	if (open_res.isErr()) {	619✔
81	log_error("unable to open file for format detection: %s -- %s",	×
82	filename.c_str(),
83	open_res.unwrapErr().c_str());
84	} else {
85	auto fd = open_res.unwrap();	619✔
86	uint8_t buffer[32];
87	auto rc = read(fd, buffer, sizeof(buffer));	619✔
88
89	if (rc < 0) {	619✔
90	log_error("unable to read file for format detection: %s -- %s",	×
91	filename.c_str(),
92	strerror(errno));
93	} else {
94	static const auto* SQLITE3_HEADER = "SQLite format 3";
95	static const auto* JAVA_CLASS_HEADER = "\xca\xfe\xba\xbe";
96
97	auto header_frag = string_fragment::from_bytes(buffer, rc);	619✔
98
99	if (header_frag.startswith(SQLITE3_HEADER)) {	619✔
100	static const auto DB_MSG
101	= lnav::console::user_message::info("SQLite database file");	1✔
102
103	log_info("%s: appears to be a SQLite DB", filename.c_str());	1✔
104	retval.dffr_file_format = file_format_t::SQLITE_DB;	1✔
105	retval.dffr_details.emplace_back(DB_MSG);	1✔
106	} else if (header_frag.startswith(JAVA_CLASS_HEADER)) {	618✔
107	static const auto CLASS_MSG = lnav::console::user_message::info(
108	"ignoring Java Class file");	×
109
110	retval.dffr_file_format = file_format_t::UNSUPPORTED;	×
111	retval.dffr_details.emplace_back(CLASS_MSG);	×
112	} else {
113	auto tf = detect_text_format(header_frag, filename);	618✔
114	auto looping = true;	618✔
115
116	switch (tf) {	618✔
117	case text_format_t::TF_UNKNOWN:	595✔
118	case text_format_t::TF_BINARY:
119	case text_format_t::TF_LOG:
120	case text_format_t::TF_JSON:
121	log_info("file does not have a known text format: %s",	595✔
122	filename.c_str());
123	break;	595✔
124	default:	23✔
125	log_info("file has text format: %s -> %d",	23✔
126	filename.c_str(),
127	tf);
128	looping = false;	23✔
129	break;	23✔
130	}
131
132	lnav::piper::multiplex_matcher mm;	1,236✔
133	file_range next_range;	618✔
134	line_buffer lb;	1,236✔
135	lb.set_fd(fd);	618✔
136
137	while (looping) {	1,135✔
138	auto load_res = lb.load_next_line(next_range);	595✔
139	if (load_res.isErr()) {	595✔
140	log_error(	×
141	"unable to load line for demux matching: %s -- %s",
142	filename.c_str(),
143	load_res.unwrapErr().c_str());
144	break;	×
145	}
146	if (!lb.is_header_utf8()) {	595✔
147	log_info("file is not UTF-8: %s", filename.c_str());	4✔
148	break;	4✔
149	}
150	if (lb.is_piper()) {	591✔
151	log_info("skipping demux match for piper file: %s",	60✔
152	filename.c_str());
153	break;	60✔
154	}
155	const auto li = load_res.unwrap();	531✔
156	if (li.li_partial) {	531✔
157	log_info("skipping demux match for partial line");	14✔
158	break;	14✔
159	}
160	auto read_res = lb.read_range(li.li_file_range);	517✔
161	if (read_res.isErr()) {	517✔
162	log_error(	×
163	"unable to read line for demux matching: %s -- %s",
164	filename.c_str(),
165	read_res.unwrapErr().c_str());
166	break;	×
167	}
168	auto sbr = read_res.unwrap();	517✔
169	auto match_res = mm.match(sbr.to_string_fragment());	517✔
170
171	looping = match_res.match(	1,034✔
172	[&retval, &filename](	×
173	lnav::piper::multiplex_matcher::found_regex f) {
174	log_info("%s: is multiplexed using pattern %s",	7✔
175	filename.c_str(),
176	f.f_id.c_str());
177	retval.dffr_file_format	7✔
178	= file_format_t::MULTIPLEXED;	7✔
179	return false;	7✔
180	},
181	[&retval, &filename](	517✔
182	lnav::piper::multiplex_matcher::found_json f) {
183	log_info("%s: is multiplexed using JSON %s",	3✔
184	filename.c_str(),
185	f.fj_id.c_str());
186	retval.dffr_file_format	3✔
187	= file_format_t::MULTIPLEXED;	3✔
188	return false;	3✔
189	},
190	[](lnav::piper::multiplex_matcher::not_found nf) {	×
191	return false;	507✔
192	},
193	[](lnav::piper::multiplex_matcher::partial p) {	×
194	return true;	×
195	});
196
197	next_range = li.li_file_range;	517✔
198	}	595✔
199	retval.dffr_details = std::move(mm.mm_details);	618✔
200	}
201	}
202	}	619✔
203
204	return retval;	619✔
205	}	619✔

tstack / lnav / 19553931481-2697

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous