17589970077-2502

Committed 09 Sep 2025 05:00PM UTC coverage: 65.196% (-5.0%) from 70.225%

Build # 17589970077-2502

Build Type

push

github

Committed by

tstack

Commit Message

[format] add fields for source file/line

Knowing the source file/line context in a log
message can help find log messages when using
log2src.

Run Details

56 of 70 new or added lines in 2 files covered. (80.0%)

13954 existing lines in 210 files now uncovered.

45516 of 69814 relevant lines covered (65.2%)

404154.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.95

/src/regexp_vtab.cc

/**
 * Copyright (c) 2017, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "base/lnav.console.into.hh"
#include "base/lnav_log.hh"
#include "column_namer.hh"
#include "config.h"
#include "lnav_util.hh"
#include "pcrepp/pcre2pp.hh"
#include "scn/scan.h"
#include "sql_help.hh"
#include "sql_util.hh"
#include "sqlitepp.hh"
#include "vtab_module.hh"
#include "yajlpp/yajlpp.hh"
#include "yajlpp/yajlpp_def.hh"

namespace {

enum {
    RC_COL_MATCH_INDEX,
    RC_COL_INDEX,
    RC_COL_NAME,
    RC_COL_CAPTURE_COUNT,
    RC_COL_RANGE_START,
    RC_COL_RANGE_STOP,
    RC_COL_CONTENT,
    RC_COL_VALUE,
    RC_COL_PATTERN,
};

struct regexp_capture {
    static constexpr const char* NAME = "regexp_capture";
    static constexpr const char* CREATE_STMT = R"(
-- The regexp_capture() table-valued function allows you to execute a regular-
-- expression over a given string and get the captured data as rows in a table.
CREATE TABLE regexp_capture (
    match_index INTEGER,
    capture_index INTEGER,
    capture_name TEXT,
    capture_count INTEGER,
    range_start INTEGER,
    range_stop INTEGER,
    content TEXT,
    value TEXT HIDDEN,
    pattern TEXT HIDDEN
);
)";

    struct cursor {
        sqlite3_vtab_cursor base;
        std::shared_ptr<lnav::pcre2pp::code> c_pattern;
        lnav::pcre2pp::match_data c_match_data{
            lnav::pcre2pp::match_data::unitialized()};
        std::string c_content;
        string_fragment c_remaining;
        bool c_content_as_blob{false};
        int c_index{0};
        bool c_matched{false};
        int c_match_index{0};
        sqlite3_int64 c_rowid{0};

        cursor(sqlite3_vtab* vt) : base({vt}) {}

        int reset() { return SQLITE_OK; }

        int next()
        {
            if (this->c_index >= (int) (this->c_match_data.get_count() - 1)) {
                auto match_res = this->c_pattern->capture_from(this->c_content)
                                     .at(this->c_remaining)
                                     .into(this->c_match_data)
                                     .matches(PCRE2_NO_UTF_CHECK)
                                     .ignore_error();
                if (match_res) {
                    this->c_remaining = match_res->f_remaining;
                }
                this->c_matched = match_res.has_value();
                this->c_index = -1;
                this->c_match_index += 1;
            }

            if (!this->c_matched) {
                return SQLITE_OK;
            }

            this->c_index += 1;

            return SQLITE_OK;
        }

        int eof() { return this->c_pattern == nullptr || !this->c_matched; }

        int get_rowid(sqlite3_int64& rowid_out)
        {
            rowid_out = this->c_rowid;

            return SQLITE_OK;
        }
    };

    int get_column(const cursor& vc, sqlite3_context* ctx, int col)
    {
        const auto cap = vc.c_match_data[vc.c_index];

        switch (col) {
            case RC_COL_MATCH_INDEX:
                sqlite3_result_int64(ctx, vc.c_match_index);
                break;
            case RC_COL_INDEX:
                sqlite3_result_int64(ctx, vc.c_index);
                break;
            case RC_COL_NAME:
                if (vc.c_index == 0) {
                    sqlite3_result_null(ctx);
                } else {
                    to_sqlite(ctx,
                              vc.c_pattern->get_name_for_capture(vc.c_index));
                }
                break;
            case RC_COL_CAPTURE_COUNT:
                sqlite3_result_int64(ctx, vc.c_match_data.get_count());
                break;
            case RC_COL_RANGE_START:
                if (cap.has_value()) {
                    sqlite3_result_int64(ctx, cap->sf_begin + 1);
                } else {
                    sqlite3_result_int64(ctx, 0);
                }
                break;
            case RC_COL_RANGE_STOP:
                if (cap.has_value()) {
                    sqlite3_result_int64(ctx, cap->sf_end + 1);
                } else {
                    sqlite3_result_int64(ctx, 0);
                }
                break;
            case RC_COL_CONTENT:
                if (cap.has_value()) {
                    to_sqlite(ctx, cap.value());
                } else {
                    sqlite3_result_null(ctx);
                }
                break;
            case RC_COL_VALUE:
                if (vc.c_content_as_blob) {
                    sqlite3_result_blob64(ctx,
                                          vc.c_content.c_str(),
                                          vc.c_content.length(),
                                          SQLITE_STATIC);
                } else {
                    sqlite3_result_text(ctx,
                                        vc.c_content.c_str(),
                                        vc.c_content.length(),
                                        SQLITE_STATIC);
                }
                break;
            case RC_COL_PATTERN: {
                to_sqlite(ctx, vc.c_pattern->get_pattern());
                break;
            }
        }

        return SQLITE_OK;
    }
};

static int
rcBestIndex(sqlite3_vtab* tab, sqlite3_index_info* pIdxInfo)
{
    vtab_index_constraints vic(pIdxInfo);
    vtab_index_usage viu(pIdxInfo);

    for (auto iter = vic.begin(); iter != vic.end(); ++iter) {
        if (iter->op != SQLITE_INDEX_CONSTRAINT_EQ) {
            continue;
        }

        switch (iter->iColumn) {
            case RC_COL_VALUE:
            case RC_COL_PATTERN:
                viu.column_used(iter);
                break;
        }
    }

    viu.allocate_args(RC_COL_VALUE, RC_COL_PATTERN, 2);
    return SQLITE_OK;
}

static int
rcFilter(sqlite3_vtab_cursor* pVtabCursor,
         int idxNum,
         const char* idxStr,
         int argc,
         sqlite3_value** argv)
{
    regexp_capture::cursor* pCur = (regexp_capture::cursor*) pVtabCursor;

    if (argc != 2) {
        pCur->c_content.clear();
        pCur->c_pattern.reset();
        return SQLITE_OK;
    }

    auto byte_count = sqlite3_value_bytes(argv[0]);
    auto blob = (const char*) sqlite3_value_blob(argv[0]);

    pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB);
    pCur->c_content.assign(blob, byte_count);

    auto pattern = from_sqlite<string_fragment>()(argc, argv, 1);
    auto compile_res = lnav::pcre2pp::code::from(pattern);
    if (compile_res.isErr()) {
        static const intern_string_t PATTERN_SRC
            = intern_string::lookup("pattern");

        set_vtable_errmsg(pVtabCursor->pVtab,
                          lnav::console::to_user_message(
                              PATTERN_SRC, compile_res.unwrapErr()));
        return SQLITE_ERROR;
    }

    pCur->c_pattern = compile_res.unwrap().to_shared();

    pCur->c_index = 0;
    pCur->c_match_data = pCur->c_pattern->create_match_data();

    pCur->c_remaining.clear();
    auto match_res = pCur->c_pattern->capture_from(pCur->c_content)
                         .into(pCur->c_match_data)
                         .matches(PCRE2_NO_UTF_CHECK)
                         .ignore_error();
    if (match_res) {
        pCur->c_remaining = match_res->f_remaining;
    }
    pCur->c_matched = match_res.has_value();
    pCur->c_match_index = 0;

    return SQLITE_OK;
}

enum {
    RCJ_COL_MATCH_INDEX,
    RCJ_COL_CONTENT,
    RCJ_COL_VALUE,
    RCJ_COL_PATTERN,
    RCJ_COL_FLAGS,
};

struct regexp_capture_flags {
    bool convert_numbers{true};
};

const typed_json_path_container<regexp_capture_flags>&
get_regexp_capture_flags_handlers()
{
    static const typed_json_path_container<regexp_capture_flags> retval
        = typed_json_path_container<regexp_capture_flags>{
            yajlpp::property_handler("convert-numbers")
                .for_field(&regexp_capture_flags::convert_numbers),
        };

    return retval;
}

struct regexp_capture_into_json {
    static constexpr const char* NAME = "regexp_capture_into_json";
    static constexpr const char* CREATE_STMT = R"(
-- The regexp_capture_into_json() table-valued function allows you to execute a
-- regular-expression over a given string and get the captured data as rows in
-- a table.
CREATE TABLE regexp_capture_into_json (
    match_index INTEGER,
    content TEXT,
    value TEXT HIDDEN,
    pattern TEXT HIDDEN,
    flags TEXT HIDDEN
);
)";

    struct cursor {
        sqlite3_vtab_cursor base;
        std::shared_ptr<lnav::pcre2pp::code> c_pattern;
        lnav::pcre2pp::match_data c_match_data{
            lnav::pcre2pp::match_data::unitialized()};
        std::unique_ptr<column_namer> c_namer;
        std::string c_content;
        string_fragment c_remaining;
        bool c_content_as_blob{false};
        bool c_matched{false};
        size_t c_match_index{0};
        sqlite3_int64 c_rowid{0};
        std::string c_flags_string;
        std::optional<regexp_capture_flags> c_flags;

        cursor(sqlite3_vtab* vt) : base({vt}) {}

        int reset() { return SQLITE_OK; }

        int next()
        {
            auto match_res = this->c_pattern->capture_from(this->c_content)
                                 .at(this->c_remaining)
                                 .into(this->c_match_data)
                                 .matches(PCRE2_NO_UTF_CHECK)
                                 .ignore_error();
            if (match_res) {
                this->c_remaining = match_res->f_remaining;
            }
            this->c_matched = match_res.has_value();
            this->c_match_index += 1;

            if (!this->c_matched) {
                return SQLITE_OK;
            }

            return SQLITE_OK;
        }

        int eof() { return this->c_pattern == nullptr || !this->c_matched; }

        int get_rowid(sqlite3_int64& rowid_out)
        {
            rowid_out = this->c_rowid;

            return SQLITE_OK;
        }
    };

    int get_column(const cursor& vc, sqlite3_context* ctx, int col)
    {
        switch (col) {
            case RCJ_COL_MATCH_INDEX:
                sqlite3_result_int64(ctx, vc.c_match_index);
                break;
            case RCJ_COL_CONTENT: {
                yajlpp_gen gen;
                yajl_gen_config(gen, yajl_gen_beautify, false);

                {
                    yajlpp_map root_map(gen);

                    for (size_t lpc = 1; lpc < vc.c_match_data.get_count();
                         lpc++)
                    {
                        const auto& colname = vc.c_namer->cn_names[lpc];
                        const auto cap = vc.c_match_data[lpc];

                        if (!cap) {
                            continue;
                        }

                        yajl_gen_pstring(gen, colname.data(), colname.length());

                        if (!vc.c_flags || vc.c_flags->convert_numbers) {
                            auto cap_view = cap->to_string_view();
                            auto scan_int_res
                                = scn::scan_int<int64_t>(cap_view, 0);

                            if (scan_int_res && scan_int_res->range().empty()) {
                                yajl_gen_integer(gen, scan_int_res->value());
                                continue;
                            }

                            auto scan_float_res
                                = scn::scan_value<double>(cap_view);
                            if (scan_float_res
                                && scan_float_res->range().empty())
                            {
                                yajl_gen_number(
                                    gen, cap_view.data(), cap_view.length());
                                continue;
                            }

                            yajl_gen_pstring(
                                gen, cap_view.data(), cap_view.length());
                        } else {
                            yajl_gen_pstring(gen, cap->data(), cap->length());
                        }
                    }
                }

                auto sf = gen.to_string_fragment();
                sqlite3_result_text(
                    ctx, sf.data(), sf.length(), SQLITE_TRANSIENT);
                sqlite3_result_subtype(ctx, JSON_SUBTYPE);
                break;
            }
            case RCJ_COL_VALUE:
                if (vc.c_content_as_blob) {
                    sqlite3_result_blob64(ctx,
                                          vc.c_content.c_str(),
                                          vc.c_content.length(),
                                          SQLITE_STATIC);
                } else {
                    sqlite3_result_text(ctx,
                                        vc.c_content.c_str(),
                                        vc.c_content.length(),
                                        SQLITE_STATIC);
                }
                break;
            case RCJ_COL_PATTERN: {
                to_sqlite(ctx, vc.c_pattern->get_pattern());
                break;
            }
            case RCJ_COL_FLAGS: {
                if (!vc.c_flags) {
                    sqlite3_result_null(ctx);
                } else {
                    to_sqlite(ctx, vc.c_flags_string);
                }
                break;
            }
        }

        return SQLITE_OK;
    }
};

static int
rcjBestIndex(sqlite3_vtab* tab, sqlite3_index_info* pIdxInfo)
{
    vtab_index_constraints vic(pIdxInfo);
    vtab_index_usage viu(pIdxInfo);

    for (auto iter = vic.begin(); iter != vic.end(); ++iter) {
        if (iter->op != SQLITE_INDEX_CONSTRAINT_EQ) {
            continue;
        }

        switch (iter->iColumn) {
            case RCJ_COL_VALUE:
            case RCJ_COL_PATTERN:
            case RCJ_COL_FLAGS:
                viu.column_used(iter);
                break;
        }
    }

    viu.allocate_args(RCJ_COL_VALUE, RCJ_COL_FLAGS, 2);
    return SQLITE_OK;
}

static int
rcjFilter(sqlite3_vtab_cursor* pVtabCursor,
          int idxNum,
          const char* idxStr,
          int argc,
          sqlite3_value** argv)
{
    auto* pCur = (regexp_capture_into_json::cursor*) pVtabCursor;

    if (argc < 2 || argc > 3) {
        pCur->c_content.clear();
        pCur->c_pattern.reset();
        pCur->c_flags_string.clear();
        pCur->c_flags = std::nullopt;
        return SQLITE_OK;
    }

    auto byte_count = sqlite3_value_bytes(argv[0]);
    const auto* blob = (const char*) sqlite3_value_blob(argv[0]);

    pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB);
    pCur->c_content.assign(blob, byte_count);

    auto pattern = from_sqlite<string_fragment>()(argc, argv, 1);
    auto compile_res = lnav::pcre2pp::code::from(pattern);
    if (compile_res.isErr()) {
        static const intern_string_t PATTERN_SRC
            = intern_string::lookup("pattern");

        set_vtable_errmsg(pVtabCursor->pVtab,
                          lnav::console::to_user_message(
                              PATTERN_SRC, compile_res.unwrapErr()));
        return SQLITE_ERROR;
    }

    pCur->c_flags_string.clear();
    pCur->c_flags = std::nullopt;
    if (argc == 3) {
        static const intern_string_t FLAGS_SRC = intern_string::lookup("flags");
        const auto flags_json = from_sqlite<string_fragment>()(argc, argv, 2);

        if (!flags_json.empty()) {
            const auto parse_res
                = get_regexp_capture_flags_handlers().parser_for(FLAGS_SRC).of(
                    flags_json);

            if (parse_res.isErr()) {
                const auto um = lnav::console::user_message::error(
                                    "unable to parse flags")
                                    .with_reason(parse_res.unwrapErr()[0])
                                    .move();

                set_vtable_errmsg(pVtabCursor->pVtab, um);
                return SQLITE_ERROR;
            }

            pCur->c_flags_string = flags_json.to_string();
            pCur->c_flags = parse_res.unwrap();
        }
    }

    pCur->c_pattern = compile_res.unwrap().to_shared();
    pCur->c_namer
        = std::make_unique<column_namer>(column_namer::language::JSON);
    pCur->c_namer->add_column("__all__"_frag);
    for (size_t lpc = 1; lpc <= pCur->c_pattern->get_capture_count(); lpc++) {
        pCur->c_namer->add_column(string_fragment::from_c_str(
            pCur->c_pattern->get_name_for_capture(lpc)));
    }

    pCur->c_match_data = pCur->c_pattern->create_match_data();
    pCur->c_remaining.clear();
    auto match_res = pCur->c_pattern->capture_from(pCur->c_content)
                         .into(pCur->c_match_data)
                         .matches(PCRE2_NO_UTF_CHECK)
                         .ignore_error();
    if (match_res) {
        pCur->c_remaining = match_res->f_remaining;
    }
    pCur->c_matched = match_res.has_value();
    pCur->c_match_index = 0;

    return SQLITE_OK;
}

}  // namespace

int
register_regexp_vtab(sqlite3* db)
{
    static vtab_module<tvt_no_update<regexp_capture>> REGEXP_CAPTURE_MODULE;
    static help_text regexp_capture_help
        = help_text("regexp_capture",
                    "A table-valued function that executes a "
                    "regular-expression over a "
                    "string and returns the captured values.  If the regex "
                    "only matches a "
                    "subset of the input string, it will be rerun on the "
                    "remaining parts "
                    "of the string until no more matches are found.")
              .sql_table_valued_function()
              .with_parameter(
                  {"string", "The string to match against the given pattern."})
              .with_parameter({"pattern", "The regular expression to match."})
              .with_result({
                  "match_index",
                  "The match iteration.  This value will increase "
                  "each time a new match is found in the input string.",
              })
              .with_result(
                  {"capture_index", "The index of the capture in the regex."})
              .with_result(
                  {"capture_name", "The name of the capture in the regex."})
              .with_result({"capture_count",
                            "The total number of captures in the regex."})
              .with_result({"range_start",
                            "The start of the capture in the input string."})
              .with_result({"range_stop",
                            "The stop of the capture in the input string."})
              .with_result({"content", "The captured value from the string."})
              .with_tags({"string"})
              .with_example({
                  "To extract the key/value pairs 'a'/1 and 'b'/2 "
                  "from the string 'a=1; b=2'",
                  "SELECT * FROM regexp_capture('a=1; b=2', "
                  "'(\\w+)=(\\d+)')",
              });

    int rc;

    REGEXP_CAPTURE_MODULE.vm_module.xBestIndex = rcBestIndex;
    REGEXP_CAPTURE_MODULE.vm_module.xFilter = rcFilter;

    rc = REGEXP_CAPTURE_MODULE.create(db, "regexp_capture");
    sqlite_function_help.insert(
        std::make_pair("regexp_capture", &regexp_capture_help));
    regexp_capture_help.index_tags();

    ensure(rc == SQLITE_OK);

    static vtab_module<tvt_no_update<regexp_capture_into_json>>
        REGEXP_CAPTURE_INTO_JSON_MODULE;
    static help_text regexp_capture_into_json_help
        = help_text(
              "regexp_capture_into_json",
              "A table-valued function that executes a "
              "regular-expression over a string and returns the captured "
              "values as a JSON object.  If the regex only matches a "
              "subset of the input string, it will be rerun on the "
              "remaining parts of the string until no more matches are found.")
              .sql_table_valued_function()
              .with_parameter(
                  {"string", "The string to match against the given pattern."})
              .with_parameter({"pattern", "The regular expression to match."})
              .with_parameter(help_text{
                  "options",
                  "A JSON object with the following option: "
                  "convert-numbers - True (default) if text that looks like "
                  "numeric data should be converted to JSON numbers, "
                  "false if they should be captured as strings."}
                                  .optional())
              .with_result({
                  "match_index",
                  "The match iteration.  This value will increase "
                  "each time a new match is found in the input string.",
              })
              .with_result({"content", "The captured values from the string."})
              .with_tags({"string"})
              .with_example({
                  "To extract the key/value pairs 'a'/1 and 'b'/2 "
                  "from the string 'a=1; b=2'",
                  "SELECT * FROM regexp_capture_into_json('a=1; b=2', "
                  "'(\\w+)=(\\d+)')",
              });

    REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xBestIndex = rcjBestIndex;
    REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xFilter = rcjFilter;

    rc = REGEXP_CAPTURE_INTO_JSON_MODULE.create(db, "regexp_capture_into_json");
    sqlite_function_help.insert(std::make_pair("regexp_capture_into_json",
                                               &regexp_capture_into_json_help));
    regexp_capture_into_json_help.index_tags();

    ensure(rc == SQLITE_OK);

    return rc;
}

1	/**
2	* Copyright (c) 2017, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*/
29
30	#include "base/lnav.console.into.hh"
31	#include "base/lnav_log.hh"
32	#include "column_namer.hh"
33	#include "config.h"
34	#include "lnav_util.hh"
35	#include "pcrepp/pcre2pp.hh"
36	#include "scn/scan.h"
37	#include "sql_help.hh"
38	#include "sql_util.hh"
39	#include "sqlitepp.hh"
40	#include "vtab_module.hh"
41	#include "yajlpp/yajlpp.hh"
42	#include "yajlpp/yajlpp_def.hh"
43
44	namespace {
45
46	enum {
47	RC_COL_MATCH_INDEX,
48	RC_COL_INDEX,
49	RC_COL_NAME,
50	RC_COL_CAPTURE_COUNT,
51	RC_COL_RANGE_START,
52	RC_COL_RANGE_STOP,
53	RC_COL_CONTENT,
54	RC_COL_VALUE,
55	RC_COL_PATTERN,
56	};
57
58	struct regexp_capture {
59	static constexpr const char* NAME = "regexp_capture";
60	static constexpr const char* CREATE_STMT = R"(
61	-- The regexp_capture() table-valued function allows you to execute a regular-
62	-- expression over a given string and get the captured data as rows in a table.
63	CREATE TABLE regexp_capture (
64	match_index INTEGER,
65	capture_index INTEGER,
66	capture_name TEXT,
67	capture_count INTEGER,
68	range_start INTEGER,
69	range_stop INTEGER,
70	content TEXT,
71	value TEXT HIDDEN,
72	pattern TEXT HIDDEN
73	);
74	)";
75
76	struct cursor {
77	sqlite3_vtab_cursor base;
78	std::shared_ptr<lnav::pcre2pp::code> c_pattern;
79	lnav::pcre2pp::match_data c_match_data{
80	lnav::pcre2pp::match_data::unitialized()};
81	std::string c_content;
82	string_fragment c_remaining;
83	bool c_content_as_blob{false};
84	int c_index{0};
85	bool c_matched{false};
86	int c_match_index{0};
87	sqlite3_int64 c_rowid{0};
88
89	cursor(sqlite3_vtab* vt) : base({vt}) {}	19✔
90
UNCOV 91	int reset() { return SQLITE_OK; }	×
92
93	int next()	71✔
94	{
95	if (this->c_index >= (int) (this->c_match_data.get_count() - 1)) {	71✔
96	auto match_res = this->c_pattern->capture_from(this->c_content)	37✔
97	.at(this->c_remaining)	37✔
98	.into(this->c_match_data)	37✔
99	.matches(PCRE2_NO_UTF_CHECK)	74✔
100	.ignore_error();	37✔
101	if (match_res) {	37✔
102	this->c_remaining = match_res->f_remaining;	20✔
103	}
104	this->c_matched = match_res.has_value();	37✔
105	this->c_index = -1;	37✔
106	this->c_match_index += 1;	37✔
107	}
108
109	if (!this->c_matched) {	71✔
110	return SQLITE_OK;	17✔
111	}
112
113	this->c_index += 1;	54✔
114
115	return SQLITE_OK;	54✔
116	}
117
118	int eof() { return this->c_pattern == nullptr \|\| !this->c_matched; }	90✔
119
UNCOV 120	int get_rowid(sqlite3_int64& rowid_out)	×
121	{
UNCOV 122	rowid_out = this->c_rowid;	×
123
124	return SQLITE_OK;	×
125	}
126	};
127
128	int get_column(const cursor& vc, sqlite3_context* ctx, int col)	509✔
129	{
130	const auto cap = vc.c_match_data[vc.c_index];	509✔
131
132	switch (col) {	509✔
133	case RC_COL_MATCH_INDEX:	50✔
134	sqlite3_result_int64(ctx, vc.c_match_index);	50✔
135	break;	50✔
136	case RC_COL_INDEX:	58✔
137	sqlite3_result_int64(ctx, vc.c_index);	58✔
138	break;	58✔
139	case RC_COL_NAME:	50✔
140	if (vc.c_index == 0) {	50✔
141	sqlite3_result_null(ctx);	20✔
142	} else {
143	to_sqlite(ctx,	30✔
144	vc.c_pattern->get_name_for_capture(vc.c_index));	30✔
145	}
146	break;	50✔
147	case RC_COL_CAPTURE_COUNT:	50✔
148	sqlite3_result_int64(ctx, vc.c_match_data.get_count());	50✔
149	break;	50✔
150	case RC_COL_RANGE_START:	51✔
151	if (cap.has_value()) {	51✔
152	sqlite3_result_int64(ctx, cap->sf_begin + 1);	50✔
153	} else {
154	sqlite3_result_int64(ctx, 0);	1✔
155	}
156	break;	51✔
157	case RC_COL_RANGE_STOP:	50✔
158	if (cap.has_value()) {	50✔
159	sqlite3_result_int64(ctx, cap->sf_end + 1);	49✔
160	} else {
161	sqlite3_result_int64(ctx, 0);	1✔
162	}
163	break;	50✔
164	case RC_COL_CONTENT:	66✔
165	if (cap.has_value()) {	66✔
166	to_sqlite(ctx, cap.value());	65✔
167	} else {
168	sqlite3_result_null(ctx);	1✔
169	}
170	break;	66✔
171	case RC_COL_VALUE:	67✔
172	if (vc.c_content_as_blob) {	67✔
173	sqlite3_result_blob64(ctx,	13✔
174	vc.c_content.c_str(),	13✔
175	vc.c_content.length(),	13✔
176	SQLITE_STATIC);
177	} else {
178	sqlite3_result_text(ctx,	54✔
179	vc.c_content.c_str(),
180	vc.c_content.length(),	54✔
181	SQLITE_STATIC);
182	}
183	break;	67✔
184	case RC_COL_PATTERN: {	67✔
185	to_sqlite(ctx, vc.c_pattern->get_pattern());	67✔
186	break;	67✔
187	}
188	}
189
190	return SQLITE_OK;	509✔
191	}
192	};
193
194	static int
195	rcBestIndex(sqlite3_vtab* tab, sqlite3_index_info* pIdxInfo)	21✔
196	{
197	vtab_index_constraints vic(pIdxInfo);	21✔
198	vtab_index_usage viu(pIdxInfo);	21✔
199
200	for (auto iter = vic.begin(); iter != vic.end(); ++iter) {	61✔
201	if (iter->op != SQLITE_INDEX_CONSTRAINT_EQ) {	40✔
UNCOV 202	continue;	×
203	}
204
205	switch (iter->iColumn) {	40✔
206	case RC_COL_VALUE:	37✔
207	case RC_COL_PATTERN:
208	viu.column_used(iter);	37✔
209	break;	37✔
210	}
211	}
212
213	viu.allocate_args(RC_COL_VALUE, RC_COL_PATTERN, 2);	21✔
214	return SQLITE_OK;	21✔
215	}
216
217	static int
218	rcFilter(sqlite3_vtab_cursor* pVtabCursor,	21✔
219	int idxNum,
220	const char* idxStr,
221	int argc,
222	sqlite3_value** argv)
223	{
224	regexp_capture::cursor* pCur = (regexp_capture::cursor*) pVtabCursor;	21✔
225
226	if (argc != 2) {	21✔
227	pCur->c_content.clear();	2✔
228	pCur->c_pattern.reset();	2✔
229	return SQLITE_OK;	2✔
230	}
231
232	auto byte_count = sqlite3_value_bytes(argv[0]);	19✔
233	auto blob = (const char*) sqlite3_value_blob(argv[0]);	19✔
234
235	pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB);	19✔
236	pCur->c_content.assign(blob, byte_count);	19✔
237
238	auto pattern = from_sqlite<string_fragment>()(argc, argv, 1);	19✔
239	auto compile_res = lnav::pcre2pp::code::from(pattern);	19✔
240	if (compile_res.isErr()) {	19✔
241	static const intern_string_t PATTERN_SRC
242	= intern_string::lookup("pattern");	6✔
243
244	set_vtable_errmsg(pVtabCursor->pVtab,	2✔
245	lnav::console::to_user_message(	4✔
246	PATTERN_SRC, compile_res.unwrapErr()));	4✔
247	return SQLITE_ERROR;	2✔
248	}
249
250	pCur->c_pattern = compile_res.unwrap().to_shared();	17✔
251
252	pCur->c_index = 0;	17✔
253	pCur->c_match_data = pCur->c_pattern->create_match_data();	17✔
254
255	pCur->c_remaining.clear();	17✔
256	auto match_res = pCur->c_pattern->capture_from(pCur->c_content)	17✔
257	.into(pCur->c_match_data)	17✔
258	.matches(PCRE2_NO_UTF_CHECK)	34✔
259	.ignore_error();	17✔
260	if (match_res) {	17✔
261	pCur->c_remaining = match_res->f_remaining;	17✔
262	}
263	pCur->c_matched = match_res.has_value();	17✔
264	pCur->c_match_index = 0;	17✔
265
266	return SQLITE_OK;	17✔
267	}	19✔
268
269	enum {
270	RCJ_COL_MATCH_INDEX,
271	RCJ_COL_CONTENT,
272	RCJ_COL_VALUE,
273	RCJ_COL_PATTERN,
274	RCJ_COL_FLAGS,
275	};
276
277	struct regexp_capture_flags {
278	bool convert_numbers{true};
279	};
280
281	const typed_json_path_container<regexp_capture_flags>&
282	get_regexp_capture_flags_handlers()	2✔
283	{
284	static const typed_json_path_container<regexp_capture_flags> retval
285	= typed_json_path_container<regexp_capture_flags>{
286	yajlpp::property_handler("convert-numbers")	4✔
287	.for_field(&regexp_capture_flags::convert_numbers),	2✔
288	};	8✔
289
290	return retval;	2✔
291	}	4✔
292
293	struct regexp_capture_into_json {
294	static constexpr const char* NAME = "regexp_capture_into_json";
295	static constexpr const char* CREATE_STMT = R"(
296	-- The regexp_capture_into_json() table-valued function allows you to execute a
297	-- regular-expression over a given string and get the captured data as rows in
298	-- a table.
299	CREATE TABLE regexp_capture_into_json (
300	match_index INTEGER,
301	content TEXT,
302	value TEXT HIDDEN,
303	pattern TEXT HIDDEN,
304	flags TEXT HIDDEN
305	);
306	)";
307
308	struct cursor {
309	sqlite3_vtab_cursor base;
310	std::shared_ptr<lnav::pcre2pp::code> c_pattern;
311	lnav::pcre2pp::match_data c_match_data{
312	lnav::pcre2pp::match_data::unitialized()};
313	std::unique_ptr<column_namer> c_namer;
314	std::string c_content;
315	string_fragment c_remaining;
316	bool c_content_as_blob{false};
317	bool c_matched{false};
318	size_t c_match_index{0};
319	sqlite3_int64 c_rowid{0};
320	std::string c_flags_string;
321	std::optional<regexp_capture_flags> c_flags;
322
323	cursor(sqlite3_vtab* vt) : base({vt}) {}	14✔
324
UNCOV 325	int reset() { return SQLITE_OK; }	×
326
327	int next()	46✔
328	{
329	auto match_res = this->c_pattern->capture_from(this->c_content)	46✔
330	.at(this->c_remaining)	46✔
331	.into(this->c_match_data)	46✔
332	.matches(PCRE2_NO_UTF_CHECK)	92✔
333	.ignore_error();	46✔
334	if (match_res) {	46✔
335	this->c_remaining = match_res->f_remaining;	30✔
336	}
337	this->c_matched = match_res.has_value();	46✔
338	this->c_match_index += 1;	46✔
339
340	if (!this->c_matched) {	46✔
341	return SQLITE_OK;	16✔
342	}
343
344	return SQLITE_OK;	30✔
345	}
346
347	int eof() { return this->c_pattern == nullptr \|\| !this->c_matched; }	64✔
348
UNCOV 349	int get_rowid(sqlite3_int64& rowid_out)	×
350	{
UNCOV 351	rowid_out = this->c_rowid;	×
352
UNCOV 353	return SQLITE_OK;	×
354	}
355	};
356
357	int get_column(const cursor& vc, sqlite3_context* ctx, int col)	189✔
358	{
359	switch (col) {	189✔
360	case RCJ_COL_MATCH_INDEX:	50✔
361	sqlite3_result_int64(ctx, vc.c_match_index);	50✔
362	break;	50✔
363	case RCJ_COL_CONTENT: {	46✔
364	yajlpp_gen gen;	46✔
365	yajl_gen_config(gen, yajl_gen_beautify, false);	46✔
366
367	{
368	yajlpp_map root_map(gen);	46✔
369
370	for (size_t lpc = 1; lpc < vc.c_match_data.get_count();	108✔
371	lpc++)
372	{
373	const auto& colname = vc.c_namer->cn_names[lpc];	62✔
374	const auto cap = vc.c_match_data[lpc];	62✔
375
376	if (!cap) {	62✔
377	continue;	19✔
378	}
379
380	yajl_gen_pstring(gen, colname.data(), colname.length());	62✔
381
382	if (!vc.c_flags \|\| vc.c_flags->convert_numbers) {	62✔
383	auto cap_view = cap->to_string_view();	60✔
384	auto scan_int_res
385	= scn::scan_int<int64_t>(cap_view, 0);	60✔
386
387	if (scan_int_res && scan_int_res->range().empty()) {	60✔
388	yajl_gen_integer(gen, scan_int_res->value());	19✔
389	continue;	19✔
390	}
391
392	auto scan_float_res
393	= scn::scan_value<double>(cap_view);	41✔
394	if (scan_float_res	41✔
395	&& scan_float_res->range().empty())	41✔
396	{
UNCOV 397	yajl_gen_number(	×
398	gen, cap_view.data(), cap_view.length());
UNCOV 399	continue;	×
400	}
401
402	yajl_gen_pstring(	41✔
403	gen, cap_view.data(), cap_view.length());
404	} else {
405	yajl_gen_pstring(gen, cap->data(), cap->length());	2✔
406	}
407	}
408	}	46✔
409
410	auto sf = gen.to_string_fragment();	46✔
411	sqlite3_result_text(	46✔
412	ctx, sf.data(), sf.length(), SQLITE_TRANSIENT);
413	sqlite3_result_subtype(ctx, JSON_SUBTYPE);	46✔
414	break;	46✔
415	}	46✔
416	case RCJ_COL_VALUE:	46✔
417	if (vc.c_content_as_blob) {	46✔
418	sqlite3_result_blob64(ctx,	×
419	vc.c_content.c_str(),	×
UNCOV 420	vc.c_content.length(),	×
421	SQLITE_STATIC);
422	} else {
423	sqlite3_result_text(ctx,	46✔
424	vc.c_content.c_str(),
425	vc.c_content.length(),	46✔
426	SQLITE_STATIC);
427	}
428	break;	46✔
429	case RCJ_COL_PATTERN: {	46✔
430	to_sqlite(ctx, vc.c_pattern->get_pattern());	46✔
431	break;	46✔
432	}
433	case RCJ_COL_FLAGS: {	1✔
434	if (!vc.c_flags) {	1✔
UNCOV 435	sqlite3_result_null(ctx);	×
436	} else {
437	to_sqlite(ctx, vc.c_flags_string);	1✔
438	}
439	break;	1✔
440	}
441	}
442
443	return SQLITE_OK;	189✔
444	}
445	};
446
447	static int
448	rcjBestIndex(sqlite3_vtab* tab, sqlite3_index_info* pIdxInfo)	15✔
449	{
450	vtab_index_constraints vic(pIdxInfo);	15✔
451	vtab_index_usage viu(pIdxInfo);	15✔
452
453	for (auto iter = vic.begin(); iter != vic.end(); ++iter) {	46✔
454	if (iter->op != SQLITE_INDEX_CONSTRAINT_EQ) {	31✔
UNCOV 455	continue;	×
456	}
457
458	switch (iter->iColumn) {	31✔
459	case RCJ_COL_VALUE:	31✔
460	case RCJ_COL_PATTERN:
461	case RCJ_COL_FLAGS:
462	viu.column_used(iter);	31✔
463	break;	31✔
464	}
465	}
466
467	viu.allocate_args(RCJ_COL_VALUE, RCJ_COL_FLAGS, 2);	15✔
468	return SQLITE_OK;	15✔
469	}
470
471	static int
472	rcjFilter(sqlite3_vtab_cursor* pVtabCursor,	20✔
473	int idxNum,
474	const char* idxStr,
475	int argc,
476	sqlite3_value** argv)
477	{
478	auto* pCur = (regexp_capture_into_json::cursor*) pVtabCursor;	20✔
479
480	if (argc < 2 \|\| argc > 3) {	20✔
481	pCur->c_content.clear();	×
482	pCur->c_pattern.reset();	×
483	pCur->c_flags_string.clear();	×
484	pCur->c_flags = std::nullopt;	×
UNCOV 485	return SQLITE_OK;	×
486	}
487
488	auto byte_count = sqlite3_value_bytes(argv[0]);	20✔
489	const auto* blob = (const char*) sqlite3_value_blob(argv[0]);	20✔
490
491	pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB);	20✔
492	pCur->c_content.assign(blob, byte_count);	20✔
493
494	auto pattern = from_sqlite<string_fragment>()(argc, argv, 1);	20✔
495	auto compile_res = lnav::pcre2pp::code::from(pattern);	20✔
496	if (compile_res.isErr()) {	20✔
497	static const intern_string_t PATTERN_SRC
498	= intern_string::lookup("pattern");	3✔
499
500	set_vtable_errmsg(pVtabCursor->pVtab,	1✔
501	lnav::console::to_user_message(	2✔
502	PATTERN_SRC, compile_res.unwrapErr()));	2✔
503	return SQLITE_ERROR;	1✔
504	}
505
506	pCur->c_flags_string.clear();	19✔
507	pCur->c_flags = std::nullopt;	19✔
508	if (argc == 3) {	19✔
509	static const intern_string_t FLAGS_SRC = intern_string::lookup("flags");	6✔
510	const auto flags_json = from_sqlite<string_fragment>()(argc, argv, 2);	2✔
511
512	if (!flags_json.empty()) {	2✔
513	const auto parse_res
514	= get_regexp_capture_flags_handlers().parser_for(FLAGS_SRC).of(	4✔
515	flags_json);	2✔
516
517	if (parse_res.isErr()) {	2✔
518	const auto um = lnav::console::user_message::error(	2✔
519	"unable to parse flags")
520	.with_reason(parse_res.unwrapErr()[0])	2✔
521	.move();	1✔
522
523	set_vtable_errmsg(pVtabCursor->pVtab, um);	1✔
524	return SQLITE_ERROR;	1✔
525	}	1✔
526
527	pCur->c_flags_string = flags_json.to_string();	1✔
528	pCur->c_flags = parse_res.unwrap();	1✔
529	}	2✔
530	}
531
532	pCur->c_pattern = compile_res.unwrap().to_shared();	18✔
533	pCur->c_namer
534	= std::make_unique<column_namer>(column_namer::language::JSON);	18✔
535	pCur->c_namer->add_column("__all__"_frag);	18✔
536	for (size_t lpc = 1; lpc <= pCur->c_pattern->get_capture_count(); lpc++) {	46✔
537	pCur->c_namer->add_column(string_fragment::from_c_str(	56✔
538	pCur->c_pattern->get_name_for_capture(lpc)));	28✔
539	}
540
541	pCur->c_match_data = pCur->c_pattern->create_match_data();	18✔
542	pCur->c_remaining.clear();	18✔
543	auto match_res = pCur->c_pattern->capture_from(pCur->c_content)	18✔
544	.into(pCur->c_match_data)	18✔
545	.matches(PCRE2_NO_UTF_CHECK)	36✔
546	.ignore_error();	18✔
547	if (match_res) {	18✔
548	pCur->c_remaining = match_res->f_remaining;	16✔
549	}
550	pCur->c_matched = match_res.has_value();	18✔
551	pCur->c_match_index = 0;	18✔
552
553	return SQLITE_OK;	18✔
554	}	20✔
555
556	} // namespace
557
558	int
559	register_regexp_vtab(sqlite3* db)	875✔
560	{
561	static vtab_module<tvt_no_update<regexp_capture>> REGEXP_CAPTURE_MODULE;	875✔
562	static help_text regexp_capture_help
563	= help_text("regexp_capture",	875✔
564	"A table-valued function that executes a "
565	"regular-expression over a "
566	"string and returns the captured values. If the regex "
567	"only matches a "
568	"subset of the input string, it will be rerun on the "
569	"remaining parts "
570	"of the string until no more matches are found.")
571	.sql_table_valued_function()	875✔
572	.with_parameter(	1,750✔
573	{"string", "The string to match against the given pattern."})
574	.with_parameter({"pattern", "The regular expression to match."})	1,750✔
575	.with_result({	1,750✔
576	"match_index",
577	"The match iteration. This value will increase "
578	"each time a new match is found in the input string.",
579	})
580	.with_result(	1,750✔
581	{"capture_index", "The index of the capture in the regex."})
582	.with_result(	1,750✔
583	{"capture_name", "The name of the capture in the regex."})
584	.with_result({"capture_count",	1,750✔
585	"The total number of captures in the regex."})
586	.with_result({"range_start",	1,750✔
587	"The start of the capture in the input string."})
588	.with_result({"range_stop",	1,750✔
589	"The stop of the capture in the input string."})
590	.with_result({"content", "The captured value from the string."})	1,750✔
591	.with_tags({"string"})	875✔
592	.with_example({	1,750✔
593	"To extract the key/value pairs 'a'/1 and 'b'/2 "
594	"from the string 'a=1; b=2'",
595	"SELECT * FROM regexp_capture('a=1; b=2', "
596	"'(\\w+)=(\\d+)')",
597	});	1,750✔
598
599	int rc;
600
601	REGEXP_CAPTURE_MODULE.vm_module.xBestIndex = rcBestIndex;	875✔
602	REGEXP_CAPTURE_MODULE.vm_module.xFilter = rcFilter;	875✔
603
604	rc = REGEXP_CAPTURE_MODULE.create(db, "regexp_capture");	875✔
605	sqlite_function_help.insert(	875✔
606	std::make_pair("regexp_capture", &regexp_capture_help));	875✔
607	regexp_capture_help.index_tags();	875✔
608
609	ensure(rc == SQLITE_OK);	875✔
610
611	static vtab_module<tvt_no_update<regexp_capture_into_json>>
612	REGEXP_CAPTURE_INTO_JSON_MODULE;	875✔
613	static help_text regexp_capture_into_json_help
614	= help_text(	875✔
615	"regexp_capture_into_json",
616	"A table-valued function that executes a "
617	"regular-expression over a string and returns the captured "
618	"values as a JSON object. If the regex only matches a "
619	"subset of the input string, it will be rerun on the "
620	"remaining parts of the string until no more matches are found.")
621	.sql_table_valued_function()	875✔
622	.with_parameter(	1,750✔
623	{"string", "The string to match against the given pattern."})
624	.with_parameter({"pattern", "The regular expression to match."})	1,750✔
625	.with_parameter(help_text{	2,625✔
626	"options",
627	"A JSON object with the following option: "
628	"convert-numbers - True (default) if text that looks like "
629	"numeric data should be converted to JSON numbers, "
630	"false if they should be captured as strings."}
631	.optional())	875✔
632	.with_result({	1,750✔
633	"match_index",
634	"The match iteration. This value will increase "
635	"each time a new match is found in the input string.",
636	})
637	.with_result({"content", "The captured values from the string."})	1,750✔
638	.with_tags({"string"})	875✔
639	.with_example({	1,750✔
640	"To extract the key/value pairs 'a'/1 and 'b'/2 "
641	"from the string 'a=1; b=2'",
642	"SELECT * FROM regexp_capture_into_json('a=1; b=2', "
643	"'(\\w+)=(\\d+)')",
644	});	1,750✔
645
646	REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xBestIndex = rcjBestIndex;	875✔
647	REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xFilter = rcjFilter;	875✔
648
649	rc = REGEXP_CAPTURE_INTO_JSON_MODULE.create(db, "regexp_capture_into_json");	875✔
650	sqlite_function_help.insert(std::make_pair("regexp_capture_into_json",	875✔
651	&regexp_capture_into_json_help));	875✔
652	regexp_capture_into_json_help.index_tags();	875✔
653
654	ensure(rc == SQLITE_OK);	875✔
655
656	return rc;	875✔
657	}

tstack / lnav / 17589970077-2502

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous