11918362462-1766

Committed 19 Nov 2024 05:22PM UTC coverage: 70.222% (-0.01%) from 70.232%

Build # 11918362462-1766

Build Type

push

github

Committed by

tstack

Commit Message

[tests] update expected output

Run Details

46302 of 65937 relevant lines covered (70.22%)

467211.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.76

/src/file_format.cc

/**
 * Copyright (c) 2020, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file file_format.hh
 */

#include "file_format.hh"

#include "archive_manager.hh"
#include "base/auto_fd.hh"
#include "base/fs_util.hh"
#include "base/intern_string.hh"
#include "base/lnav_log.hh"
#include "config.h"
#include "line_buffer.hh"
#include "text_format.hh"

detect_file_format_result
detect_file_format(const std::filesystem::path& filename)
{
    detect_file_format_result retval = {file_format_t::UNKNOWN};
    auto describe_res = archive_manager::describe(filename);
    if (describe_res.isOk()
        && describe_res.unwrap().is<archive_manager::archive_info>())
    {
        return {file_format_t::ARCHIVE};
    }

    auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);
    if (open_res.isErr()) {
        log_error("unable to open file for format detection: %s -- %s",
                  filename.c_str(),
                  open_res.unwrapErr().c_str());
    } else {
        auto fd = open_res.unwrap();
        uint8_t buffer[32];
        auto rc = read(fd, buffer, sizeof(buffer));

        if (rc < 0) {
            log_error("unable to read file for format detection: %s -- %s",
                      filename.c_str(),
                      strerror(errno));
        } else {
            static auto SQLITE3_HEADER = "SQLite format 3";
            auto header_frag = string_fragment::from_bytes(buffer, rc);

            if (header_frag.startswith(SQLITE3_HEADER)) {
                log_info("%s: appears to be a SQLite DB", filename.c_str());
                retval.dffr_file_format = file_format_t::SQLITE_DB;
            } else {
                auto tf = detect_text_format(header_frag, filename);
                auto looping = true;

                switch (tf) {
                    case text_format_t::TF_UNKNOWN:
                    case text_format_t::TF_BINARY:
                    case text_format_t::TF_LOG:
                        log_info("file does not have a known text format: %s",
                                 filename.c_str());
                        break;
                    default:
                        log_info("file has text format: %s -> %d",
                                 filename.c_str(),
                                 tf);
                        looping = false;
                        break;
                }

                lnav::piper::multiplex_matcher mm;
                file_range next_range;
                line_buffer lb;
                lb.set_fd(fd);

                while (looping) {
                    auto load_res = lb.load_next_line(next_range);
                    if (load_res.isErr()) {
                        log_error(
                            "unable to load line for demux matching: %s -- %s",
                            filename.c_str(),
                            load_res.unwrapErr().c_str());
                        break;
                    }
                    if (!lb.is_header_utf8()) {
                        log_info("file is not UTF-8: %s", filename.c_str());
                        break;
                    }
                    if (lb.is_piper()) {
                        log_info("skipping demux match for piper file: %s",
                                 filename.c_str());
                        break;
                    }
                    const auto li = load_res.unwrap();
                    if (li.li_partial) {
                        log_info("skipping demux match for partial line");
                        break;
                    }
                    auto read_res = lb.read_range(li.li_file_range);
                    if (read_res.isErr()) {
                        log_error(
                            "unable to read line for demux matching: %s -- %s",
                            filename.c_str(),
                            read_res.unwrapErr().c_str());
                        break;
                    }
                    auto sbr = read_res.unwrap();
                    auto match_res = mm.match(sbr.to_string_fragment());

                    looping = match_res.match(
                        [&retval,
                         &filename](lnav::piper::multiplex_matcher::found f) {
                            log_info("%s: is multiplexed using %s",
                                     filename.c_str(),
                                     f.f_id.c_str());
                            retval.dffr_file_format
                                = file_format_t::MULTIPLEXED;
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::not_found nf) {
                            return false;
                        },
                        [](lnav::piper::multiplex_matcher::partial p) {
                            return true;
                        });

                    next_range = li.li_file_range;
                }
                retval.dffr_details = std::move(mm.mm_details);
            }
        }
    }

    return retval;
}

1	/**
2	* Copyright (c) 2020, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file file_format.hh
30	*/
31
32	#include "file_format.hh"
33
34	#include "archive_manager.hh"
35	#include "base/auto_fd.hh"
36	#include "base/fs_util.hh"
37	#include "base/intern_string.hh"
38	#include "base/lnav_log.hh"
39	#include "config.h"
40	#include "line_buffer.hh"
41	#include "text_format.hh"
42
43	detect_file_format_result
44	detect_file_format(const std::filesystem::path& filename)	507✔
45	{
46	detect_file_format_result retval = {file_format_t::UNKNOWN};	507✔
47	auto describe_res = archive_manager::describe(filename);	507✔
48	if (describe_res.isOk()	1,014✔
49	&& describe_res.unwrap().is<archive_manager::archive_info>())	507✔
50	{
51	return {file_format_t::ARCHIVE};	×
52	}
53
54	auto open_res = lnav::filesystem::open_file(filename, O_RDONLY);	507✔
55	if (open_res.isErr()) {	507✔
56	log_error("unable to open file for format detection: %s -- %s",	×
57	filename.c_str(),
58	open_res.unwrapErr().c_str());
59	} else {
60	auto fd = open_res.unwrap();	507✔
61	uint8_t buffer[32];
62	auto rc = read(fd, buffer, sizeof(buffer));	507✔
63
64	if (rc < 0) {	507✔
65	log_error("unable to read file for format detection: %s -- %s",	×
66	filename.c_str(),
67	strerror(errno));
68	} else {
69	static auto SQLITE3_HEADER = "SQLite format 3";
70	auto header_frag = string_fragment::from_bytes(buffer, rc);	507✔
71
72	if (header_frag.startswith(SQLITE3_HEADER)) {	507✔
73	log_info("%s: appears to be a SQLite DB", filename.c_str());	1✔
74	retval.dffr_file_format = file_format_t::SQLITE_DB;	1✔
75	} else {
76	auto tf = detect_text_format(header_frag, filename);	506✔
77	auto looping = true;	506✔
78
79	switch (tf) {	506✔
80	case text_format_t::TF_UNKNOWN:	434✔
81	case text_format_t::TF_BINARY:
82	case text_format_t::TF_LOG:
83	log_info("file does not have a known text format: %s",	434✔
84	filename.c_str());
85	break;	434✔
86	default:	72✔
87	log_info("file has text format: %s -> %d",	72✔
88	filename.c_str(),
89	tf);
90	looping = false;	72✔
91	break;	72✔
92	}
93
94	lnav::piper::multiplex_matcher mm;	1,012✔
95	file_range next_range;	506✔
96	line_buffer lb;	1,012✔
97	lb.set_fd(fd);	506✔
98
99	while (looping) {	898✔
100	auto load_res = lb.load_next_line(next_range);	434✔
101	if (load_res.isErr()) {	434✔
102	log_error(	×
103	"unable to load line for demux matching: %s -- %s",
104	filename.c_str(),
105	load_res.unwrapErr().c_str());
106	break;	×
107	}
108	if (!lb.is_header_utf8()) {	434✔
109	log_info("file is not UTF-8: %s", filename.c_str());	3✔
110	break;	3✔
111	}
112	if (lb.is_piper()) {	431✔
113	log_info("skipping demux match for piper file: %s",	38✔
114	filename.c_str());
115	break;	38✔
116	}
117	const auto li = load_res.unwrap();	393✔
118	if (li.li_partial) {	393✔
119	log_info("skipping demux match for partial line");	1✔
120	break;	1✔
121	}
122	auto read_res = lb.read_range(li.li_file_range);	392✔
123	if (read_res.isErr()) {	392✔
124	log_error(	×
125	"unable to read line for demux matching: %s -- %s",
126	filename.c_str(),
127	read_res.unwrapErr().c_str());
128	break;	×
129	}
130	auto sbr = read_res.unwrap();	392✔
131	auto match_res = mm.match(sbr.to_string_fragment());	392✔
132
133	looping = match_res.match(	392✔
134	[&retval,	5✔
135	&filename](lnav::piper::multiplex_matcher::found f) {	10✔
136	log_info("%s: is multiplexed using %s",	5✔
137	filename.c_str(),
138	f.f_id.c_str());
139	retval.dffr_file_format
140	= file_format_t::MULTIPLEXED;	5✔
141	return false;	5✔
142	},
143	[](lnav::piper::multiplex_matcher::not_found nf) {	387✔
144	return false;	387✔
145	},
146	[](lnav::piper::multiplex_matcher::partial p) {	×
147	return true;	×
148	});
149
150	next_range = li.li_file_range;	392✔
151	}	434✔
152	retval.dffr_details = std::move(mm.mm_details);	506✔
153	}
154	}
155	}	507✔
156
157	return retval;	507✔
158	}	507✔

tstack / lnav / 11918362462-1766

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous