29185833209

Committed 12 Jul 2026 08:25AM UTC coverage: 51.235% (-1.5%) from 52.754%

Build # 29185833209

Build Type

Pull #96

github

Committed by

web-flow

Commit Message

Merge 0b9f07110 into 056c79287

Pull Request Pull Request #96: fix: consolidate stale-index rebuilds across consumers and fix multi-run breaks

Coverage Stats

33724 of 84486 branches covered (39.92%)

Branch coverage included in aggregate %.

140 of 147 new or added lines in 15 files covered. (95.24%)

5188 existing lines in 197 files now uncovered.

34547 of 48765 relevant lines covered (70.84%)

10791.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.23

/src/dftracer/utils/utilities/reader/trace_reader_prefilter.cpp

#include <dftracer/utils/utilities/common/query/query.h>
#include <dftracer/utils/utilities/reader/internal/trace_reader_prefilter.h>
#include <simdjson.h>

#include <cstring>
#include <optional>
#include <string>
#include <type_traits>
#include <vector>

namespace dftracer::utils::utilities::reader::internal {

using common::query::Query;

namespace {

bool collect_and_eq_literals(const common::query::QueryNode& node,
                             std::vector<std::string>& out) {
    return std::visit(
        [&out](const auto& n) -> bool {
            using T = std::decay_t<decltype(n)>;
            if constexpr (std::is_same_v<T, common::query::CompareNode>) {
                if (n.op != common::query::CompareOp::EQ) return false;
                std::string lit;
                lit.reserve(n.field.path.size() + 16);
                lit += '"';
                lit += n.field.path;
                lit += "\":";
                const auto& val = n.value.value;
                if (std::holds_alternative<std::string>(val)) {
                    lit += '"';
                    lit += std::get<std::string>(val);
                    lit += '"';
                } else if (std::holds_alternative<int64_t>(val)) {
                    lit += std::to_string(std::get<int64_t>(val));
                } else if (std::holds_alternative<uint64_t>(val)) {
                    lit += std::to_string(std::get<uint64_t>(val));
                } else if (std::holds_alternative<bool>(val)) {
                    lit += std::get<bool>(val) ? "true" : "false";
                } else {
                    return false;  // double or other: skip pre-filter
                }
                out.push_back(std::move(lit));
                return true;
            } else if constexpr (std::is_same_v<T, common::query::AndNode>) {
                return collect_and_eq_literals(*n.left, out) &&
                       collect_and_eq_literals(*n.right, out);
            }
            return false;  // OrNode, NotNode, InNode, NotInNode, CompareNode
                           // with non-EQ op: conservative skip
        },
        node.data);
}

// Top-level JSON keys in dftracer events. Anything else in the query DSL
// (e.g. `epoch == 0`, `fhash == "..."`) refers to a field nested under
// "args"; the same convention collect_query_fields relies on when it
// folds nested object keys into the flat ValueMap.
bool is_top_level_event_key(std::string_view k) {
    return k == "id" || k == "name" || k == "cat" || k == "pid" || k == "tid" ||
           k == "ts" || k == "dur" || k == "ph";
}

// Walk a CompareNode-with-EQ leaf into a probe. Returns false on
// unsupported shapes (more than one '.' or a literal type the simdjson
// get_X path can't compare directly).
bool compile_eq_leaf(const common::query::CompareNode& n,
                     CompiledEqProbe& out) {
    if (n.op != common::query::CompareOp::EQ) return false;
    auto dot = n.field.path.find('.');
    if (dot == std::string::npos) {
        if (is_top_level_event_key(n.field.path)) {
            out.top_key = n.field.path;
            out.nested_key.clear();
        } else {
            // Bare arg-style key: foo -> args.foo.
            out.top_key = "args";
            out.nested_key = n.field.path;
        }
    } else {
        if (n.field.path.find('.', dot + 1) != std::string::npos) return false;
        out.top_key = n.field.path.substr(0, dot);
        out.nested_key = n.field.path.substr(dot + 1);
    }
    return std::visit(
        [&out](auto&& v) -> bool {
            using T = std::decay_t<decltype(v)>;
            if constexpr (std::is_same_v<T, std::string>) {
                out.kind = CompiledEqProbe::Kind::String;
                out.s_val = v;
                return true;
            } else if constexpr (std::is_same_v<T, std::int64_t>) {
                out.kind = CompiledEqProbe::Kind::Int64;
                out.i64_val = v;
                return true;
            } else if constexpr (std::is_same_v<T, std::uint64_t>) {
                out.kind = CompiledEqProbe::Kind::UInt64;
                out.u64_val = v;
                return true;
            } else if constexpr (std::is_same_v<T, double>) {
                out.kind = CompiledEqProbe::Kind::Double;
                out.d_val = v;
                return true;
            } else if constexpr (std::is_same_v<T, bool>) {
                out.kind = CompiledEqProbe::Kind::Bool;
                out.b_val = v;
                return true;
            } else {
                return false;
            }
        },
        n.value.value);
}

bool probe_matches_value(const CompiledEqProbe& p,
                         simdjson::ondemand::value val) {
    switch (p.kind) {
        case CompiledEqProbe::Kind::String: {
            auto r = val.get_string();
            if (r.error()) return false;
            auto sv = r.value_unsafe();
            return sv.size() == p.s_val.size() &&
                   std::memcmp(sv.data(), p.s_val.data(), sv.size()) == 0;
        }
        case CompiledEqProbe::Kind::Int64: {
            auto t = val.type();
            if (t.error()) return false;
            if (t.value_unsafe() == simdjson::ondemand::json_type::number) {
                auto num = val.get_number();
                if (num.error()) return false;
                auto n = num.value_unsafe();
                if (n.is_int64()) return n.get_int64() == p.i64_val;
                if (n.is_uint64()) {
                    if (p.i64_val < 0) return false;
                    return n.get_uint64() ==
                           static_cast<std::uint64_t>(p.i64_val);
                }
                return n.get_double() == static_cast<double>(p.i64_val);
            }
            return false;
        }
        case CompiledEqProbe::Kind::UInt64: {
            auto num = val.get_number();
            if (num.error()) return false;
            auto n = num.value_unsafe();
            if (n.is_uint64()) return n.get_uint64() == p.u64_val;
            if (n.is_int64()) {
                auto v = n.get_int64();
                if (v < 0) return false;
                return static_cast<std::uint64_t>(v) == p.u64_val;
            }
            return n.get_double() == static_cast<double>(p.u64_val);
        }
        case CompiledEqProbe::Kind::Double: {
            auto r = val.get_double();
            if (r.error()) return false;
            return r.value_unsafe() == p.d_val;
        }
        case CompiledEqProbe::Kind::Bool: {
            auto r = val.get_bool();
            if (r.error()) return false;
            return r.value_unsafe() == p.b_val;
        }
    }
    return false;
}

}  // namespace

// Try to compile the query AST as an AND of EQ leaves. nullopt on
// unsupported shapes; the ValueMap path handles those.
std::optional<std::vector<CompiledEqProbe>> try_compile_eq_probes(
    const common::query::QueryNode& node) {
    using namespace common::query;
    return std::visit(
        [&](const auto& n) -> std::optional<std::vector<CompiledEqProbe>> {
            using T = std::decay_t<decltype(n)>;
            if constexpr (std::is_same_v<T, CompareNode>) {
                CompiledEqProbe p;
                if (!compile_eq_leaf(n, p)) return std::nullopt;
                return std::vector<CompiledEqProbe>{std::move(p)};
            } else if constexpr (std::is_same_v<T, AndNode>) {
                auto l = try_compile_eq_probes(*n.left);
                if (!l) return std::nullopt;
                auto r = try_compile_eq_probes(*n.right);
                if (!r) return std::nullopt;
                l->insert(l->end(), std::make_move_iterator(r->begin()),
                          std::make_move_iterator(r->end()));
                return l;
            } else {
                return std::nullopt;
            }
        },
        node.data);
}

// Evaluate compiled AND-of-EQ probes by directly probing simdjson fields.
bool eval_compiled_eq(const std::vector<CompiledEqProbe>& probes,
                      simdjson::ondemand::document_reference doc) {
    for (const auto& p : probes) {
        doc.rewind();
        auto top_r = doc.find_field_unordered(
            std::string_view(p.top_key.data(), p.top_key.size()));
        if (top_r.error()) return false;
        auto top_v = top_r.value();
        if (p.nested_key.empty()) {
            if (!probe_matches_value(p, top_v)) return false;
        } else {
            auto obj_r = top_v.get_object();
            if (obj_r.error()) return false;
            auto inner_r = obj_r.value().find_field_unordered(
                std::string_view(p.nested_key.data(), p.nested_key.size()));
            if (inner_r.error()) return false;
            if (!probe_matches_value(p, inner_r.value())) return false;
        }
    }
    return true;
}

LinePrefilter build_prefilter(const Query& q) {
    // Short literals like `"pid":1000` or `"epoch":0` are common enough in
    // practice that memmem on every line costs more than it saves on the
    // parse side. Only keep literals long enough that rarity is plausible
    // (hashes, filenames, host names).
    constexpr std::size_t MIN_LITERAL_LEN = 16;

    LinePrefilter pf;
    std::vector<std::string> tmp;
    if (collect_and_eq_literals(q.root(), tmp)) {
        for (auto& lit : tmp) {
            if (lit.size() >= MIN_LITERAL_LEN) {
                pf.required.push_back(std::move(lit));
            }
        }
    }
    return pf;
}

}  // namespace dftracer::utils::utilities::reader::internal

1	#include <dftracer/utils/utilities/common/query/query.h>
2	#include <dftracer/utils/utilities/reader/internal/trace_reader_prefilter.h>
3	#include <simdjson.h>
4
5	#include <cstring>
6	#include <optional>
7	#include <string>
8	#include <type_traits>
9	#include <vector>
10
11	namespace dftracer::utils::utilities::reader::internal {
12
13	using common::query::Query;
14
15	namespace {
16
17	bool collect_and_eq_literals(const common::query::QueryNode& node,	47✔
18	std::vector<std::string>& out) {
19	return std::visit(	47✔
20	[&out](const auto& n) -> bool {	94✔
21	using T = std::decay_t<decltype(n)>;
22	if constexpr (std::is_same_v<T, common::query::CompareNode>) {
23	if (n.op != common::query::CompareOp::EQ) return false;	18!
24	std::string lit;	18✔
25	lit.reserve(n.field.path.size() + 16);	18!
26	lit += '"';	18!
27	lit += n.field.path;	18!
28	lit += "\":";	18!
29	const auto& val = n.value.value;	18✔
30	if (std::holds_alternative<std::string>(val)) {	18!
31	lit += '"';	18!
32	lit += std::get<std::string>(val);	18!
33	lit += '"';	18!
34	} else if (std::holds_alternative<int64_t>(val)) {	18!
35	lit += std::to_string(std::get<int64_t>(val));	×
36	} else if (std::holds_alternative<uint64_t>(val)) {	×
37	lit += std::to_string(std::get<uint64_t>(val));	×
38	} else if (std::holds_alternative<bool>(val)) {	×
39	lit += std::get<bool>(val) ? "true" : "false";	×
UNCOV 40	} else {	×
41	return false; // double or other: skip pre-filter	×
42	}
43	out.push_back(std::move(lit));	18!
44	return true;	18✔
45	} else if constexpr (std::is_same_v<T, common::query::AndNode>) {	18✔
46	return collect_and_eq_literals(*n.left, out) &&	×
47	collect_and_eq_literals(*n.right, out);	×
48	}
49	return false; // OrNode, NotNode, InNode, NotInNode, CompareNode	29✔
50	// with non-EQ op: conservative skip
51	},	18✔
52	node.data);	47✔
53	}
54
55	// Top-level JSON keys in dftracer events. Anything else in the query DSL
56	// (e.g. `epoch == 0`, `fhash == "..."`) refers to a field nested under
57	// "args"; the same convention collect_query_fields relies on when it
58	// folds nested object keys into the flat ValueMap.
59	bool is_top_level_event_key(std::string_view k) {	7✔
60	return k == "id" \|\| k == "name" \|\| k == "cat" \|\| k == "pid" \|\| k == "tid" \|\|	7!
UNCOV 61	k == "ts" \|\| k == "dur" \|\| k == "ph";	×
62	}
63
64	// Walk a CompareNode-with-EQ leaf into a probe. Returns false on
65	// unsupported shapes (more than one '.' or a literal type the simdjson
66	// get_X path can't compare directly).
67	bool compile_eq_leaf(const common::query::CompareNode& n,	7✔
68	CompiledEqProbe& out) {
69	if (n.op != common::query::CompareOp::EQ) return false;	7!
70	auto dot = n.field.path.find('.');	7✔
71	if (dot == std::string::npos) {	7!
72	if (is_top_level_event_key(n.field.path)) {	7!
73	out.top_key = n.field.path;	7✔
74	out.nested_key.clear();	7✔
75	} else {	7✔
76	// Bare arg-style key: foo -> args.foo.
77	out.top_key = "args";	×
78	out.nested_key = n.field.path;	×
79	}
80	} else {	7✔
81	if (n.field.path.find('.', dot + 1) != std::string::npos) return false;	×
82	out.top_key = n.field.path.substr(0, dot);	×
83	out.nested_key = n.field.path.substr(dot + 1);	×
84	}
85	return std::visit(	7✔
86	[&out](auto&& v) -> bool {	14✔
87	using T = std::decay_t<decltype(v)>;
88	if constexpr (std::is_same_v<T, std::string>) {
89	out.kind = CompiledEqProbe::Kind::String;	6✔
90	out.s_val = v;	6✔
91	return true;	6✔
92	} else if constexpr (std::is_same_v<T, std::int64_t>) {
93	out.kind = CompiledEqProbe::Kind::Int64;	×
94	out.i64_val = v;	×
95	return true;	×
96	} else if constexpr (std::is_same_v<T, std::uint64_t>) {
97	out.kind = CompiledEqProbe::Kind::UInt64;	1✔
98	out.u64_val = v;	1✔
99	return true;	1✔
100	} else if constexpr (std::is_same_v<T, double>) {
101	out.kind = CompiledEqProbe::Kind::Double;	×
102	out.d_val = v;	×
103	return true;	×
104	} else if constexpr (std::is_same_v<T, bool>) {
105	out.kind = CompiledEqProbe::Kind::Bool;	×
106	out.b_val = v;	×
107	return true;	×
108	} else {
109	return false;
110	}
111	},
112	n.value.value);	7✔
113	}	7✔
114
115	bool probe_matches_value(const CompiledEqProbe& p,	×
116	simdjson::ondemand::value val) {
117	switch (p.kind) {	×
118	case CompiledEqProbe::Kind::String: {
119	auto r = val.get_string();	×
120	if (r.error()) return false;	×
121	auto sv = r.value_unsafe();	×
122	return sv.size() == p.s_val.size() &&	×
123	std::memcmp(sv.data(), p.s_val.data(), sv.size()) == 0;	×
124	}
125	case CompiledEqProbe::Kind::Int64: {
126	auto t = val.type();	×
127	if (t.error()) return false;	×
128	if (t.value_unsafe() == simdjson::ondemand::json_type::number) {	×
129	auto num = val.get_number();	×
130	if (num.error()) return false;	×
131	auto n = num.value_unsafe();	×
132	if (n.is_int64()) return n.get_int64() == p.i64_val;	×
133	if (n.is_uint64()) {	×
134	if (p.i64_val < 0) return false;	×
135	return n.get_uint64() ==	×
136	static_cast<std::uint64_t>(p.i64_val);	×
137	}
138	return n.get_double() == static_cast<double>(p.i64_val);	×
139	}
140	return false;	×
141	}
142	case CompiledEqProbe::Kind::UInt64: {
143	auto num = val.get_number();	×
144	if (num.error()) return false;	×
145	auto n = num.value_unsafe();	×
146	if (n.is_uint64()) return n.get_uint64() == p.u64_val;	×
147	if (n.is_int64()) {	×
148	auto v = n.get_int64();	×
149	if (v < 0) return false;	×
150	return static_cast<std::uint64_t>(v) == p.u64_val;	×
151	}
152	return n.get_double() == static_cast<double>(p.u64_val);	×
153	}
154	case CompiledEqProbe::Kind::Double: {
155	auto r = val.get_double();	×
156	if (r.error()) return false;	×
157	return r.value_unsafe() == p.d_val;	×
158	}
159	case CompiledEqProbe::Kind::Bool: {
160	auto r = val.get_bool();	×
161	if (r.error()) return false;	×
162	return r.value_unsafe() == p.b_val;	×
163	}
164	}
165	return false;	×
UNCOV 166	}	×
167
168	} // namespace
169
170	// Try to compile the query AST as an AND of EQ leaves. nullopt on
171	// unsupported shapes; the ValueMap path handles those.
172	std::optional<std::vector<CompiledEqProbe>> try_compile_eq_probes(	7✔
173	const common::query::QueryNode& node) {
174	using namespace common::query;
175	return std::visit(	7✔
176	[&](const auto& n) -> std::optional<std::vector<CompiledEqProbe>> {	7✔
177	using T = std::decay_t<decltype(n)>;
178	if constexpr (std::is_same_v<T, CompareNode>) {
179	CompiledEqProbe p;	7✔
180	if (!compile_eq_leaf(n, p)) return std::nullopt;	7!
181	return std::vector<CompiledEqProbe>{std::move(p)};	7!
182	} else if constexpr (std::is_same_v<T, AndNode>) {	7✔
183	auto l = try_compile_eq_probes(*n.left);	×
184	if (!l) return std::nullopt;	×
185	auto r = try_compile_eq_probes(*n.right);	×
186	if (!r) return std::nullopt;	×
187	l->insert(l->end(), std::make_move_iterator(r->begin()),	×
UNCOV 188	std::make_move_iterator(r->end()));	×
189	return l;	×
190	} else {	×
191	return std::nullopt;	×
192	}
193	},	7✔
194	node.data);	7✔
195	}
196
197	// Evaluate compiled AND-of-EQ probes by directly probing simdjson fields.
198	bool eval_compiled_eq(const std::vector<CompiledEqProbe>& probes,	×
199	simdjson::ondemand::document_reference doc) {
200	for (const auto& p : probes) {	×
201	doc.rewind();	×
202	auto top_r = doc.find_field_unordered(	×
UNCOV 203	std::string_view(p.top_key.data(), p.top_key.size()));	×
204	if (top_r.error()) return false;	×
205	auto top_v = top_r.value();	×
206	if (p.nested_key.empty()) {	×
207	if (!probe_matches_value(p, top_v)) return false;	×
UNCOV 208	} else {	×
209	auto obj_r = top_v.get_object();	×
210	if (obj_r.error()) return false;	×
211	auto inner_r = obj_r.value().find_field_unordered(	×
UNCOV 212	std::string_view(p.nested_key.data(), p.nested_key.size()));	×
213	if (inner_r.error()) return false;	×
214	if (!probe_matches_value(p, inner_r.value())) return false;	×
215	}
216	}
217	return true;	×
UNCOV 218	}	×
219
220	LinePrefilter build_prefilter(const Query& q) {	47✔
221	// Short literals like `"pid":1000` or `"epoch":0` are common enough in
222	// practice that memmem on every line costs more than it saves on the
223	// parse side. Only keep literals long enough that rarity is plausible
224	// (hashes, filenames, host names).
225	constexpr std::size_t MIN_LITERAL_LEN = 16;	47✔
226
227	LinePrefilter pf;	47✔
228	std::vector<std::string> tmp;	47✔
229	if (collect_and_eq_literals(q.root(), tmp)) {	47!
230	for (auto& lit : tmp) {	36✔
231	if (lit.size() >= MIN_LITERAL_LEN) {	18✔
232	pf.required.push_back(std::move(lit));	3!
233	}	3✔
234	}
235	}	18✔
236	return pf;	47✔
237	}	47!
238
239	} // namespace dftracer::utils::utilities::reader::internal

llnl / dftracer-utils / 29185833209

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous