11918362462-1766

Committed 19 Nov 2024 05:22PM UTC coverage: 70.222% (-0.01%) from 70.232%

Build # 11918362462-1766

Build Type

push

github

Committed by

tstack

Commit Message

[tests] update expected output

Run Details

46302 of 65937 relevant lines covered (70.22%)

467211.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.39

/src/text_format.cc

/**
 * Copyright (c) 2017, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file text_format.cc
 */

#include <set>

#include "text_format.hh"

#include "base/lnav_log.hh"
#include "config.h"
#include "pcrepp/pcre2pp.hh"
#include "yajl/api/yajl_parse.h"

text_format_t
detect_text_format(string_fragment sf,
                   std::optional<std::filesystem::path> path)
{
    static const std::set<std::filesystem::path> FILTER_EXTS = {
        ".bz2",
        ".gz",
        ".lzma",
        ".xz",
        ".zst",
    };
    static const auto C_EXTS = std::set<std::filesystem::path>{
        ".h",
        ".hh",
        ".hpp",
        ".c",
        ".cc",
        ".cpp",
        ".tpp",
    };
    static const auto PY_EXT = std::filesystem::path(".py");
    static const auto RS_EXT = std::filesystem::path(".rs");
    static const auto SQL_EXT = std::filesystem::path(".sql");
    static const auto JAVA_EXT = std::filesystem::path(".java");
    static const auto TOML_EXT = std::filesystem::path(".toml");
    static const auto XML_EXT = std::filesystem::path(".xml");
    static const auto YAML_EXT = std::filesystem::path(".yaml");
    static const auto YML_EXT = std::filesystem::path(".yml");
    static const auto MAKEFILE_STEM = std::filesystem::path("Makefile");
    static const auto MD_EXT = std::filesystem::path(".md");
    static const auto MARKDOWN_EXT = std::filesystem::path(".markdown");
    static const auto SH_EXT = std::filesystem::path(".sh");

    static const auto DIFF_MATCHERS = lnav::pcre2pp::code::from_const(
        R"(^--- .*\n\+\+\+ .*\n)", PCRE2_MULTILINE);

    static const auto MAN_MATCHERS = lnav::pcre2pp::code::from_const(
        R"(^[A-Za-z][A-Za-z\-_\+0-9]+\(\d\)\s+)", PCRE2_MULTILINE);

    // XXX This is a pretty crude way of
    // detecting format...
    static const auto PYTHON_MATCHERS = lnav::pcre2pp::code::from_const(
        "(?:"
        "^\\s*def\\s+\\w+\\([^)]*\\):"
        "[^\\n]*$|"
        "^\\s*try:[^\\n]*$"
        ")",
        PCRE2_MULTILINE);

    static const auto RUST_MATCHERS = lnav::pcre2pp::code::from_const(
        R"(
(?:
^\s*use\s+[\w+:\{\}]+;$|
^\s*(?:pub enum|pub const|(?:pub )?fn)\s+\w+.*$|
^\s*impl\s+\w+.*$
)
)",
        PCRE2_MULTILINE);

    static const auto JAVA_MATCHERS = lnav::pcre2pp::code::from_const(
        "(?:"
        "^package\\s+|"
        "^import\\s+|"
        "^\\s*(?:public)?\\s*"
        "class\\s*(\\w+\\s+)*\\s*{"
        ")",
        PCRE2_MULTILINE);

    static const auto C_LIKE_MATCHERS = lnav::pcre2pp::code::from_const(
        "(?:"
        "^#\\s*include\\s+|"
        "^#\\s*define\\s+|"
        "^\\s*if\\s+\\([^)]+\\)[^\\n]"
        "*$|"
        "^\\s*(?:\\w+\\s+)*class "
        "\\w+ {"
        ")",
        PCRE2_MULTILINE);

    static const auto SQL_MATCHERS = lnav::pcre2pp::code::from_const(
        "(?:"
        "create\\s+table\\s+|"
        "select\\s+.+\\s+from\\s+|"
        "insert\\s+into\\s+.+\\s+"
        "values"
        ")",
        PCRE2_MULTILINE | PCRE2_CASELESS);

    static const auto XML_MATCHERS = lnav::pcre2pp::code::from_const(
        "(?:"
        R"(<\?xml(\s+\w+\s*=\s*"[^"]*")*\?>|)"
        R"(</?\w+(\s+\w+\s*=\s*"[^"]*")*\s*>)"
        ")",
        PCRE2_MULTILINE | PCRE2_CASELESS);

    static const auto SH_MATCHERS
        = lnav::pcre2pp::code::from_const("^#!.+sh\\b", PCRE2_MULTILINE);

    if (path) {
        while (FILTER_EXTS.count(path->extension()) > 0) {
            path = path->stem();
        }

        auto stem = path->stem();
        auto ext = path->extension();
        if (ext == MD_EXT || ext == MARKDOWN_EXT) {
            return text_format_t::TF_MARKDOWN;
        }

        if (C_EXTS.count(ext) > 0) {
            return text_format_t::TF_C_LIKE;
        }

        if (ext == PY_EXT) {
            return text_format_t::TF_PYTHON;
        }

        if (ext == RS_EXT) {
            return text_format_t::TF_RUST;
        }

        if (ext == SQL_EXT) {
            return text_format_t::TF_SQL;
        }

        if (ext == TOML_EXT) {
            return text_format_t::TF_TOML;
        }

        if (ext == JAVA_EXT) {
            return text_format_t::TF_JAVA;
        }

        if (ext == YAML_EXT || ext == YML_EXT) {
            return text_format_t::TF_YAML;
        }

        if (ext == XML_EXT) {
            return text_format_t::TF_XML;
        }

        if (stem == MAKEFILE_STEM) {
            return text_format_t::TF_MAKEFILE;
        }

        if (stem == SH_EXT) {
            return text_format_t::TF_SHELL_SCRIPT;
        }
    }

    {
        auto_mem<yajl_handle_t> jhandle(yajl_free);

        jhandle = yajl_alloc(nullptr, nullptr, nullptr);
        if (yajl_parse(jhandle, sf.udata(), sf.length()) == yajl_status_ok) {
            return text_format_t::TF_JSON;
        }
    }

    if (DIFF_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_DIFF;
    }

    if (SH_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_SHELL_SCRIPT;
    }

    if (MAN_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_MAN;
    }

    if (PYTHON_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_PYTHON;
    }

    if (RUST_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_RUST;
    }

    if (JAVA_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_JAVA;
    }

    if (C_LIKE_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_C_LIKE;
    }

    if (SQL_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_SQL;
    }

    if (XML_MATCHERS.find_in(sf).ignore_error()) {
        return text_format_t::TF_XML;
    }

    return text_format_t::TF_UNKNOWN;
}

std::optional<text_format_meta_t>
extract_text_meta(string_fragment sf, text_format_t tf)
{
    static const auto MAN_NAME = lnav::pcre2pp::code::from_const(
        R"(^([A-Za-z][A-Za-z\-_\+0-9]+\(\d\))\s+)", PCRE2_MULTILINE);

    switch (tf) {
        case text_format_t::TF_MAN: {
            static thread_local auto md
                = lnav::pcre2pp::match_data::unitialized();

            auto find_res
                = MAN_NAME.capture_from(sf).into(md).matches().ignore_error();

            if (find_res) {
                return text_format_meta_t{
                    md.to_string(),
                };
            }
            break;
        }
        default:
            break;
    }

    return std::nullopt;
}

1	/**
2	* Copyright (c) 2017, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file text_format.cc
30	*/
31
32	#include <set>
33
34	#include "text_format.hh"
35
36	#include "base/lnav_log.hh"
37	#include "config.h"
38	#include "pcrepp/pcre2pp.hh"
39	#include "yajl/api/yajl_parse.h"
40
41	text_format_t
42	detect_text_format(string_fragment sf,	1,030✔
43	std::optional<std::filesystem::path> path)
44	{
45	static const std::set<std::filesystem::path> FILTER_EXTS = {
46	".bz2",
47	".gz",
48	".lzma",
49	".xz",
50	".zst",
51	};	3,550✔
52	static const auto C_EXTS = std::set<std::filesystem::path>{
53	".h",
54	".hh",
55	".hpp",
56	".c",
57	".cc",
58	".cpp",
59	".tpp",
60	};	4,558✔
61	static const auto PY_EXT = std::filesystem::path(".py");	1,030✔
62	static const auto RS_EXT = std::filesystem::path(".rs");	1,030✔
63	static const auto SQL_EXT = std::filesystem::path(".sql");	1,030✔
64	static const auto JAVA_EXT = std::filesystem::path(".java");	1,030✔
65	static const auto TOML_EXT = std::filesystem::path(".toml");	1,030✔
66	static const auto XML_EXT = std::filesystem::path(".xml");	1,030✔
67	static const auto YAML_EXT = std::filesystem::path(".yaml");	1,030✔
68	static const auto YML_EXT = std::filesystem::path(".yml");	1,030✔
69	static const auto MAKEFILE_STEM = std::filesystem::path("Makefile");	1,030✔
70	static const auto MD_EXT = std::filesystem::path(".md");	1,030✔
71	static const auto MARKDOWN_EXT = std::filesystem::path(".markdown");	1,030✔
72	static const auto SH_EXT = std::filesystem::path(".sh");	1,030✔
73
74	static const auto DIFF_MATCHERS = lnav::pcre2pp::code::from_const(
75	R"(^--- .\n\+\+\+ .\n)", PCRE2_MULTILINE);	1,030✔
76
77	static const auto MAN_MATCHERS = lnav::pcre2pp::code::from_const(
78	R"(^[A-Za-z][A-Za-z\-_\+0-9]+\(\d\)\s+)", PCRE2_MULTILINE);	1,030✔
79
80	// XXX This is a pretty crude way of
81	// detecting format...
82	static const auto PYTHON_MATCHERS = lnav::pcre2pp::code::from_const(
83	"(?:"
84	"^\\sdef\\s+\\w+\\([^)]\\):"
85	"[^\\n]*$\|"
86	"^\\stry:[^\\n]$"
87	")",
88	PCRE2_MULTILINE);	1,030✔
89
90	static const auto RUST_MATCHERS = lnav::pcre2pp::code::from_const(
91	R"(
92	(?:
93	^\s*use\s+[\w+:\{\}]+;$\|
94	^\s(?:pub enum\|pub const\|(?:pub )?fn)\s+\w+.$\|
95	^\simpl\s+\w+.$
96	)
97	)",
98	PCRE2_MULTILINE);	1,030✔
99
100	static const auto JAVA_MATCHERS = lnav::pcre2pp::code::from_const(
101	"(?:"
102	"^package\\s+\|"
103	"^import\\s+\|"
104	"^\\s(?:public)?\\s"
105	"class\\s(\\w+\\s+)\\s*{"
106	")",
107	PCRE2_MULTILINE);	1,030✔
108
109	static const auto C_LIKE_MATCHERS = lnav::pcre2pp::code::from_const(
110	"(?:"
111	"^#\\s*include\\s+\|"
112	"^#\\s*define\\s+\|"
113	"^\\s*if\\s+\\([^)]+\\)[^\\n]"
114	"*$\|"
115	"^\\s(?:\\w+\\s+)class "
116	"\\w+ {"
117	")",
118	PCRE2_MULTILINE);	1,030✔
119
120	static const auto SQL_MATCHERS = lnav::pcre2pp::code::from_const(
121	"(?:"
122	"create\\s+table\\s+\|"
123	"select\\s+.+\\s+from\\s+\|"
124	"insert\\s+into\\s+.+\\s+"
125	"values"
126	")",
127	PCRE2_MULTILINE \| PCRE2_CASELESS);	1,030✔
128
129	static const auto XML_MATCHERS = lnav::pcre2pp::code::from_const(
130	"(?:"
131	R"(<\?xml(\s+\w+\s=\s"[^"]")\?>\|)"
132	R"(</?\w+(\s+\w+\s=\s"[^"]")\s*>)"
133	")",
134	PCRE2_MULTILINE \| PCRE2_CASELESS);	1,030✔
135
136	static const auto SH_MATCHERS
137	= lnav::pcre2pp::code::from_const("^#!.+sh\\b", PCRE2_MULTILINE);	1,030✔
138
139	if (path) {	1,030✔
140	while (FILTER_EXTS.count(path->extension()) > 0) {	1,036✔
141	path = path->stem();	6✔
142	}
143
144	auto stem = path->stem();	1,030✔
145	auto ext = path->extension();	1,030✔
146	if (ext == MD_EXT \|\| ext == MARKDOWN_EXT) {	1,030✔
147	return text_format_t::TF_MARKDOWN;	20✔
148	}
149
150	if (C_EXTS.count(ext) > 0) {	1,010✔
151	return text_format_t::TF_C_LIKE;	2✔
152	}
153
154	if (ext == PY_EXT) {	1,008✔
155	return text_format_t::TF_PYTHON;	2✔
156	}
157
158	if (ext == RS_EXT) {	1,006✔
159	return text_format_t::TF_RUST;	×
160	}
161
162	if (ext == SQL_EXT) {	1,006✔
163	return text_format_t::TF_SQL;	×
164	}
165
166	if (ext == TOML_EXT) {	1,006✔
167	return text_format_t::TF_TOML;	×
168	}
169
170	if (ext == JAVA_EXT) {	1,006✔
171	return text_format_t::TF_JAVA;	×
172	}
173
174	if (ext == YAML_EXT \|\| ext == YML_EXT) {	1,006✔
175	return text_format_t::TF_YAML;	×
176	}
177
178	if (ext == XML_EXT) {	1,006✔
179	return text_format_t::TF_XML;	6✔
180	}
181
182	if (stem == MAKEFILE_STEM) {	1,000✔
183	return text_format_t::TF_MAKEFILE;	×
184	}
185
186	if (stem == SH_EXT) {	1,000✔
187	return text_format_t::TF_SHELL_SCRIPT;	×
188	}
189	}	1,060✔
190
191	{
192	auto_mem<yajl_handle_t> jhandle(yajl_free);	1,000✔
193
194	jhandle = yajl_alloc(nullptr, nullptr, nullptr);	1,000✔
195	if (yajl_parse(jhandle, sf.udata(), sf.length()) == yajl_status_ok) {	1,000✔
196	return text_format_t::TF_JSON;	68✔
197	}
198	}	1,000✔
199
200	if (DIFF_MATCHERS.find_in(sf).ignore_error()) {	932✔
201	return text_format_t::TF_DIFF;	1✔
202	}
203
204	if (SH_MATCHERS.find_in(sf).ignore_error()) {	931✔
205	return text_format_t::TF_SHELL_SCRIPT;	1✔
206	}
207
208	if (MAN_MATCHERS.find_in(sf).ignore_error()) {	930✔
209	return text_format_t::TF_MAN;	5✔
210	}
211
212	if (PYTHON_MATCHERS.find_in(sf).ignore_error()) {	925✔
213	return text_format_t::TF_PYTHON;	×
214	}
215
216	if (RUST_MATCHERS.find_in(sf).ignore_error()) {	925✔
217	return text_format_t::TF_RUST;	×
218	}
219
220	if (JAVA_MATCHERS.find_in(sf).ignore_error()) {	925✔
221	return text_format_t::TF_JAVA;	×
222	}
223
224	if (C_LIKE_MATCHERS.find_in(sf).ignore_error()) {	925✔
225	return text_format_t::TF_C_LIKE;	×
226	}
227
228	if (SQL_MATCHERS.find_in(sf).ignore_error()) {	925✔
229	return text_format_t::TF_SQL;	×
230	}
231
232	if (XML_MATCHERS.find_in(sf).ignore_error()) {	925✔
233	return text_format_t::TF_XML;	18✔
234	}
235
236	return text_format_t::TF_UNKNOWN;	907✔
237	}
238
239	std::optional<text_format_meta_t>
240	extract_text_meta(string_fragment sf, text_format_t tf)	75✔
241	{
242	static const auto MAN_NAME = lnav::pcre2pp::code::from_const(
243	R"(^([A-Za-z][A-Za-z\-_\+0-9]+\(\d\))\s+)", PCRE2_MULTILINE);	75✔
244
245	switch (tf) {	75✔
246	case text_format_t::TF_MAN: {	3✔
247	static thread_local auto md
248	= lnav::pcre2pp::match_data::unitialized();	3✔
249
250	auto find_res
251	= MAN_NAME.capture_from(sf).into(md).matches().ignore_error();	3✔
252
253	if (find_res) {	3✔
254	return text_format_meta_t{	6✔
255	md.to_string(),
256	};	3✔
257	}
258	break;	×
259	}
260	default:	72✔
261	break;	72✔
262	}
263
264	return std::nullopt;	72✔
265	}

tstack / lnav / 11918362462-1766

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous