19550717862-2695

Committed 20 Nov 2025 08:41PM UTC coverage: 68.863% (-0.007%) from 68.87%

Build # 19550717862-2695

Build Type

push

github

Committed by

tstack

Commit Message

[file_format] check for unsupported files

Don't waste time indexing stuff the user is
not likely to be interested in.

Run Details

4 of 18 new or added lines in 4 files covered. (22.22%)

1 existing line in 1 file now uncovered.

51083 of 74181 relevant lines covered (68.86%)

431818.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.72

/src/file_format.cc

/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file file_format.hh
 */

#include "file_format.hh"

#include "archive_manager.hh"
#include "base/auto_fd.hh"
#include "base/fs_util.hh"
#include "base/intern_string.hh"
#include "base/lnav.console.hh"
#include "base/lnav_log.hh"
#include "config.h"
#include "line_buffer.hh"
#include "piper.match.hh"
#include "text_format.hh"

detect_file_format_result
detect_file_format(const std::filesystem::path& filename)
{
    static const auto JAR_EXT = std::filesystem::path(".jar");

    // static auto op = lnav_operation{"detect_file_format"};
    // auto op_guard = lnav_opid_guard::internal(op);
    log_trace("detecting format of file: %s", filename.c_str());

    detect_file_format_result retval = {file_format_t::UNKNOWN};
    auto ext = filename.extension();

    if (ext == JAR_EXT) {
        static const auto JAR_MSG
            = lnav::console::user_message::info("ignoring Java JAR file");
        return {file_format_t::UNSUPPORTED, {JAR_MSG}};
    }

    auto describe_res = archive_manager::describe(filename);
    if (describe_res.isOk()
        && describe_res.unwrap().is<archive_manager::archive_info>())
    {
        return {file_format_t::ARCHIVE};
    }

    auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);
    if (open_res.isErr()) {
        log_error("unable to open file for format detection: %s -- %s",
                  filename.c_str(),
                  open_res.unwrapErr().c_str());
    } else {
        auto fd = open_res.unwrap();
        uint8_t buffer[32];
        auto rc = read(fd, buffer, sizeof(buffer));

        if (rc < 0) {
            log_error("unable to read file for format detection: %s -- %s",
                      filename.c_str(),
                      strerror(errno));
        } else {
            static const auto* SQLITE3_HEADER = "SQLite format 3";
            static const auto* JAVA_CLASS_HEADER = "\xca\xfe\xba\xbe";

            auto header_frag = string_fragment::from_bytes(buffer, rc);

            if (header_frag.startswith(SQLITE3_HEADER)) {
                log_info("%s: appears to be a SQLite DB", filename.c_str());
                retval.dffr_file_format = file_format_t::SQLITE_DB;
            } else if (header_frag.startswith(JAVA_CLASS_HEADER)) {
                static const auto CLASS_MSG = lnav::console::user_message::info(
                    "ignoring Java Class file");

                retval.dffr_file_format = file_format_t::UNSUPPORTED;
                retval.dffr_details.emplace_back(CLASS_MSG);
            } else {
                auto tf = detect_text_format(header_frag, filename);
                auto looping = true;

                switch (tf) {
                    case text_format_t::TF_UNKNOWN:
                    case text_format_t::TF_BINARY:
                    case text_format_t::TF_LOG:
                    case text_format_t::TF_JSON:
                        log_info("file does not have a known text format: %s",
                                 filename.c_str());
                        break;
                    default:
                        log_info("file has text format: %s -> %d",
                                 filename.c_str(),
                                 tf);
                        looping = false;
                        break;
                }

                lnav::piper::multiplex_matcher mm;
                file_range next_range;
                line_buffer lb;
                lb.set_fd(fd);

                while (looping) {
                    auto load_res = lb.load_next_line(next_range);
                    if (load_res.isErr()) {
                        log_error(
                            "unable to load line for demux matching: %s -- %s",
                            filename.c_str(),
                            load_res.unwrapErr().c_str());
                        break;
                    }
                    if (!lb.is_header_utf8()) {
                        log_info("file is not UTF-8: %s", filename.c_str());
                        break;
                    }
                    if (lb.is_piper()) {
                        log_info("skipping demux match for piper file: %s",
                                 filename.c_str());
                        break;
                    }
                    const auto li = load_res.unwrap();
                    if (li.li_partial) {
                        log_info("skipping demux match for partial line");
                        break;
                    }
                    auto read_res = lb.read_range(li.li_file_range);
                    if (read_res.isErr()) {
                        log_error(
                            "unable to read line for demux matching: %s -- %s",
                            filename.c_str(),
                            read_res.unwrapErr().c_str());
                        break;
                    }
                    auto sbr = read_res.unwrap();
                    auto match_res = mm.match(sbr.to_string_fragment());

                    looping = match_res.match(
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_regex f) {
                            log_info("%s: is multiplexed using pattern %s",
                                     filename.c_str(),
                                     f.f_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [&retval, &filename](
                            lnav::piper::multiplex_matcher::found_json f) {
                            log_info("%s: is multiplexed using JSON %s",
                                     filename.c_str(),
                                     f.fj_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::not_found nf) {
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::partial p) {
                            return true;
                        });

                    next_range = li.li_file_range;
                }
                retval.dffr_details = std::move(mm.mm_details);
            }
        }
    }

    return retval;
}

1	/**
2	* Copyright (c) 2020, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file file_format.hh
30	*/
31
32	#include "file_format.hh"
33
34	#include "archive_manager.hh"
35	#include "base/auto_fd.hh"
36	#include "base/fs_util.hh"
37	#include "base/intern_string.hh"
38	#include "base/lnav.console.hh"
39	#include "base/lnav_log.hh"
40	#include "config.h"
41	#include "line_buffer.hh"
42	#include "piper.match.hh"
43	#include "text_format.hh"
44
45	detect_file_format_result
46	detect_file_format(const std::filesystem::path& filename)	619✔
47	{
48	static const auto JAR_EXT = std::filesystem::path(".jar");	619✔
49
50	// static auto op = lnav_operation{"detect_file_format"};
51	// auto op_guard = lnav_opid_guard::internal(op);
52	log_trace("detecting format of file: %s", filename.c_str());	619✔
53
54	detect_file_format_result retval = {file_format_t::UNKNOWN};	619✔
55	auto ext = filename.extension();	619✔
56
57	if (ext == JAR_EXT) {	619✔
58	static const auto JAR_MSG
NEW 59	= lnav::console::user_message::info("ignoring Java JAR file");	×
NEW 60	return {file_format_t::UNSUPPORTED, {JAR_MSG}};	×
61	}
62
63	auto describe_res = archive_manager::describe(filename);	619✔
64	if (describe_res.isOk()	1,238✔
65	&& describe_res.unwrap().is<archive_manager::archive_info>())	619✔
66	{
67	return {file_format_t::ARCHIVE};	×
68	}
69
70	auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);	619✔
71	if (open_res.isErr()) {	619✔
72	log_error("unable to open file for format detection: %s -- %s",	×
73	filename.c_str(),
74	open_res.unwrapErr().c_str());
75	} else {
76	auto fd = open_res.unwrap();	619✔
77	uint8_t buffer[32];
78	auto rc = read(fd, buffer, sizeof(buffer));	619✔
79
80	if (rc < 0) {	619✔
81	log_error("unable to read file for format detection: %s -- %s",	×
82	filename.c_str(),
83	strerror(errno));
84	} else {
85	static const auto* SQLITE3_HEADER = "SQLite format 3";
86	static const auto* JAVA_CLASS_HEADER = "\xca\xfe\xba\xbe";
87
88	auto header_frag = string_fragment::from_bytes(buffer, rc);	619✔
89
90	if (header_frag.startswith(SQLITE3_HEADER)) {	619✔
91	log_info("%s: appears to be a SQLite DB", filename.c_str());	1✔
92	retval.dffr_file_format = file_format_t::SQLITE_DB;	1✔
93	} else if (header_frag.startswith(JAVA_CLASS_HEADER)) {	618✔
94	static const auto CLASS_MSG = lnav::console::user_message::info(
NEW 95	"ignoring Java Class file");	×
96
NEW 97	retval.dffr_file_format = file_format_t::UNSUPPORTED;	×
NEW 98	retval.dffr_details.emplace_back(CLASS_MSG);	×
99	} else {
100	auto tf = detect_text_format(header_frag, filename);	618✔
101	auto looping = true;	618✔
102
103	switch (tf) {	618✔
104	case text_format_t::TF_UNKNOWN:	595✔
105	case text_format_t::TF_BINARY:
106	case text_format_t::TF_LOG:
107	case text_format_t::TF_JSON:
108	log_info("file does not have a known text format: %s",	595✔
109	filename.c_str());
110	break;	595✔
111	default:	23✔
112	log_info("file has text format: %s -> %d",	23✔
113	filename.c_str(),
114	tf);
115	looping = false;	23✔
116	break;	23✔
117	}
118
119	lnav::piper::multiplex_matcher mm;	1,236✔
120	file_range next_range;	618✔
121	line_buffer lb;	1,236✔
122	lb.set_fd(fd);	618✔
123
124	while (looping) {	1,135✔
125	auto load_res = lb.load_next_line(next_range);	595✔
126	if (load_res.isErr()) {	595✔
127	log_error(	×
128	"unable to load line for demux matching: %s -- %s",
129	filename.c_str(),
130	load_res.unwrapErr().c_str());
131	break;	×
132	}
133	if (!lb.is_header_utf8()) {	595✔
134	log_info("file is not UTF-8: %s", filename.c_str());	4✔
135	break;	4✔
136	}
137	if (lb.is_piper()) {	591✔
138	log_info("skipping demux match for piper file: %s",	60✔
139	filename.c_str());
140	break;	60✔
141	}
142	const auto li = load_res.unwrap();	531✔
143	if (li.li_partial) {	531✔
144	log_info("skipping demux match for partial line");	14✔
145	break;	14✔
146	}
147	auto read_res = lb.read_range(li.li_file_range);	517✔
148	if (read_res.isErr()) {	517✔
149	log_error(	×
150	"unable to read line for demux matching: %s -- %s",
151	filename.c_str(),
152	read_res.unwrapErr().c_str());
153	break;	×
154	}
155	auto sbr = read_res.unwrap();	517✔
156	auto match_res = mm.match(sbr.to_string_fragment());	517✔
157
158	looping = match_res.match(	1,034✔
159	[&retval, &filename](	×
160	lnav::piper::multiplex_matcher::found_regex f) {
161	log_info("%s: is multiplexed using pattern %s",	7✔
162	filename.c_str(),
163	f.f_id.c_str());
164	retval.dffr_file_format	7✔
165	= file_format_t::MULTIPLEXED;	7✔
166	return false;	7✔
167	},
168	[&retval, &filename](	517✔
169	lnav::piper::multiplex_matcher::found_json f) {
170	log_info("%s: is multiplexed using JSON %s",	3✔
171	filename.c_str(),
172	f.fj_id.c_str());
173	retval.dffr_file_format	3✔
174	= file_format_t::MULTIPLEXED;	3✔
175	return false;	3✔
176	},
177	[](lnav::piper::multiplex_matcher::not_found nf) {	×
178	return false;	507✔
179	},
180	[](lnav::piper::multiplex_matcher::partial p) {	×
181	return true;	×
182	});
183
184	next_range = li.li_file_range;	517✔
185	}	595✔
186	retval.dffr_details = std::move(mm.mm_details);	618✔
187	}
188	}
189	}	619✔
190
191	return retval;	619✔
192	}	619✔

tstack / lnav / 19550717862-2695

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous