30067416336

Committed 24 Jul 2026 04:40AM UTC coverage: 50.693% (-2.0%) from 52.66%

Build # 30067416336

Build Type

Pull #99

github

Committed by

web-flow

Commit Message

Merge 464586f0d into 06bc84ec9

Pull Request Pull Request #99: Support CM time_metric (NS/MS/SEC/US) across reader and viz

Coverage Stats

18063 of 48203 branches covered (37.47%)

Branch coverage included in aggregate %.

1514 of 2204 new or added lines in 58 files covered. (68.69%)

741 existing lines in 114 files now uncovered.

23715 of 34210 relevant lines covered (69.32%)

39842.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.33

/src/dftracer/utils/python/utilities/statistics_query.cpp

#define PY_SSIZE_T_CLEAN
#include <dftracer/utils/core/coro/task.h>
#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/core/tasks/coro_scope.h>
#include <dftracer/utils/python/py_dict_helpers.h>
#include <dftracer/utils/python/py_runtime_mixin.h>
#include <dftracer/utils/python/py_type_helpers.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/statistics_query.h>
#include <dftracer/utils/utilities/composites/dft/indexing/resolve_and_build.h>
#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
#include <dftracer/utils/utilities/composites/dft/statistics/statistics_query_utility.h>

#include <string>

using dftracer::utils::CoroScope;
using dftracer::utils::run_coro_scope;
using dftracer::utils::Runtime;
using dftracer::utils::coro::CoroTask;
using namespace dftracer::utils::utilities::composites::dft::statistics;
namespace indexing = dftracer::utils::utilities::composites::dft::indexing;

DFTRACER_UTILS_RUNTIME_BACKED_SLOTS(StatisticsQuery, StatisticsQueryObject)

static PyObject *StatisticsQuery_query(StatisticsQueryObject *self,
                                       PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"file_path", "query_type", "top_n",
                                   "index_dir", NULL};
    const char *file_path;
    const char *query_type_str = "summary";
    Py_ssize_t top_n = 10;
    const char *index_dir = "";

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sns", (char **)kwlist,
                                     &file_path, &query_type_str, &top_n,
                                     &index_dir)) {
        return NULL;
    }

    StatisticsQueryType qt;
    if (strcmp(query_type_str, "summary") == 0) {
        qt = StatisticsQueryType::SUMMARY;
    } else if (strcmp(query_type_str, "categories") == 0) {
        qt = StatisticsQueryType::CATEGORIES;
    } else if (strcmp(query_type_str, "names") == 0) {
        qt = StatisticsQueryType::NAMES;
    } else if (strcmp(query_type_str, "pid_tids") == 0) {
        qt = StatisticsQueryType::PID_TIDS;
    } else if (strcmp(query_type_str, "time_range") == 0) {
        qt = StatisticsQueryType::TIME_RANGE;
    } else if (strcmp(query_type_str, "duration_stats") == 0) {
        qt = StatisticsQueryType::DURATION_STATS;
    } else if (strcmp(query_type_str, "top_n_names") == 0) {
        qt = StatisticsQueryType::TOP_N_NAMES;
    } else if (strcmp(query_type_str, "top_n_categories") == 0) {
        qt = StatisticsQueryType::TOP_N_CATEGORIES;
    } else if (strcmp(query_type_str, "detailed") == 0) {
        qt = StatisticsQueryType::DETAILED;
    } else {
        PyErr_Format(PyExc_ValueError, "unknown query_type: '%s'",
                     query_type_str);
        return NULL;
    }

    std::string file_path_str(file_path);
    std::string index_dir_str(index_dir);
    TraceStatistics stats;
    StatisticsQueryOutput output;
    auto qt_copy = qt;
    auto top_n_copy = static_cast<std::uint64_t>(top_n);

    if (!run_blocking([&] {
            Runtime *rt = resolve_runtime(self);

            StatisticsAggregatorInput agg_input;
            agg_input.file_path = file_path_str;
            agg_input.index_dir = index_dir_str;
            agg_input.index_path = dftracer::utils::utilities::composites::dft::
                internal::determine_index_path(file_path_str, index_dir_str);

            rt->submit(run_coro_scope(rt->executor(),
                                      [file_path_str, index_dir_str](
                                          CoroScope &scope) -> CoroTask<void> {
                                          co_await indexing::ensure_index_fresh(
                                              &scope, "", file_path_str,
                                              index_dir_str, false);
                                      }),
                       "stats-ensure-fresh")
                .get();

            auto *stats_p = &stats;
            auto agg_task = [stats_p, agg_input]() -> CoroTask<void> {
                StatisticsAggregatorUtility util;
                *stats_p = co_await util.process(agg_input);
            };
            rt->submit(agg_task(), "stats-agg").get();

            StatisticsQueryInput query_input;
            query_input.stats = std::move(stats);
            query_input.query_type = qt_copy;
            query_input.top_n = top_n_copy;

            auto *out_p = &output;
            auto query_task = [out_p, query_input]() -> CoroTask<void> {
                StatisticsQueryUtility util;
                *out_p = co_await util.process(query_input);
            };
            rt->submit(query_task(), "stats-query").get();
        })) {
        return NULL;
    }

    PyObject *results_list =
        PyList_New(static_cast<Py_ssize_t>(output.results.size()));
    if (!results_list) return NULL;

    for (std::size_t i = 0; i < output.results.size(); ++i) {
        PyObject *tup = PyTuple_New(2);
        if (!tup) {
            Py_DECREF(results_list);
            return NULL;
        }
        PyTuple_SET_ITEM(tup, 0,
                         PyUnicode_FromString(output.results[i].first.c_str()));
        PyTuple_SET_ITEM(tup, 1,
                         PyLong_FromUnsignedLongLong(output.results[i].second));
        PyList_SET_ITEM(results_list, static_cast<Py_ssize_t>(i), tup);
    }

    PyObject *d = PyDict_New();
    if (!d) {
        Py_DECREF(results_list);
        return NULL;
    }

    int rc = 0;
    rc |= dict_set_str(d, "query_type", output.query_type_name.c_str());
    rc |= dict_set_u64(d, "total_events", output.total_events);
    rc |= dict_set_u64(d, "min_timestamp_us", output.min_timestamp_us);
    rc |= dict_set_u64(d, "max_timestamp_us", output.max_timestamp_us);
    rc |= dict_set_f64(d, "time_span_seconds", output.time_span_seconds);
    rc |= dict_set_u64(d, "duration_count", output.duration_count);
    rc |= dict_set_f64(d, "duration_mean_us", output.duration_mean_us);
    rc |= dict_set_f64(d, "duration_stddev_us", output.duration_stddev_us);
    rc |= dict_set_u64(d, "duration_min_us", output.duration_min_us);
    rc |= dict_set_u64(d, "duration_max_us", output.duration_max_us);
    // Steals (and always releases) the results_list reference.
    rc |= dict_set_steal(d, "results", results_list);

    if (rc != 0) {
        Py_DECREF(d);
        return NULL;
    }

    return d;
}

static PyObject *StatisticsQuery_call(PyObject *self, PyObject *args,
                                      PyObject *kwds) {
    return StatisticsQuery_query((StatisticsQueryObject *)self, args, kwds);
}

static PyMethodDef StatisticsQuery_methods[] = {
    {"process", (PyCFunction)StatisticsQuery_query,
     METH_VARARGS | METH_KEYWORDS,
     "process(file_path, query_type='summary', top_n=10, index_dir='')\n"
     "--\n"
     "\n"
     "Query statistics from an indexed trace file.\n"
     "\n"
     "Args:\n"
     "    file_path (str): Path to the trace file.\n"
     "    query_type (str): Query type (default 'summary'). One of\n"
     "        'summary', 'categories', 'names', 'pid_tids',\n"
     "        'time_range', 'duration_stats', 'top_n_names',\n"
     "        'top_n_categories', 'detailed'.\n"
     "    top_n (int): Top results for ranked queries (default 10).\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Query results.\n"},
    {NULL}};

PyTypeObject StatisticsQueryUtilityType = {
    PyVarObject_HEAD_INIT(
        NULL, 0) "dftracer_utils_ext.StatisticsQueryUtility", /* tp_name */
    sizeof(StatisticsQueryObject),                            /* tp_basicsize */
    0,                                                        /* tp_itemsize */
    (destructor)StatisticsQuery_dealloc,                      /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    StatisticsQuery_call,                     /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "StatisticsQueryUtility(runtime: Runtime | None = None)\n"
    "--\n\n"
    "Query pre-computed statistics from an indexed trace file.\n\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n",
    0,                              /* tp_traverse */
    0,                              /* tp_clear */
    0,                              /* tp_richcompare */
    0,                              /* tp_weaklistoffset */
    0,                              /* tp_iter */
    0,                              /* tp_iternext */
    StatisticsQuery_methods,        /* tp_methods */
    0,                              /* tp_members */
    0,                              /* tp_getset */
    0,                              /* tp_base */
    0,                              /* tp_dict */
    0,                              /* tp_descr_get */
    0,                              /* tp_descr_set */
    0,                              /* tp_dictoffset */
    (initproc)StatisticsQuery_init, /* tp_init */
    0,                              /* tp_alloc */
    StatisticsQuery_new,            /* tp_new */
};

int init_statistics_query(PyObject *m) {
    if (register_type(m, &StatisticsQueryUtilityType,
                      "StatisticsQueryUtility") < 0)
        return -1;

    return 0;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/core/coro/task.h>
3	#include <dftracer/utils/core/runtime.h>
4	#include <dftracer/utils/core/tasks/coro_scope.h>
5	#include <dftracer/utils/python/py_dict_helpers.h>
6	#include <dftracer/utils/python/py_runtime_mixin.h>
7	#include <dftracer/utils/python/py_type_helpers.h>
8	#include <dftracer/utils/python/runtime.h>
9	#include <dftracer/utils/python/utilities/statistics_query.h>
10	#include <dftracer/utils/utilities/composites/dft/indexing/resolve_and_build.h>
11	#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
12	#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
13	#include <dftracer/utils/utilities/composites/dft/statistics/statistics_query_utility.h>
14
15	#include <string>
16
17	using dftracer::utils::CoroScope;
18	using dftracer::utils::run_coro_scope;
19	using dftracer::utils::Runtime;
20	using dftracer::utils::coro::CoroTask;
21	using namespace dftracer::utils::utilities::composites::dft::statistics;
22	namespace indexing = dftracer::utils::utilities::composites::dft::indexing;
23
24	DFTRACER_UTILS_RUNTIME_BACKED_SLOTS(StatisticsQuery, StatisticsQueryObject)	24✔
25
26	static PyObject StatisticsQuery_query(StatisticsQueryObject self,	8✔
27	PyObject args, PyObject kwds) {
28	static const char *kwlist[] = {"file_path", "query_type", "top_n",
29	"index_dir", NULL};
30	const char *file_path;
31	const char *query_type_str = "summary";	8✔
32	Py_ssize_t top_n = 10;	8✔
33	const char *index_dir = "";	8✔
34
35	if (!PyArg_ParseTupleAndKeywords(args, kwds, "s\|sns", (char **)kwlist,	8!
36	&file_path, &query_type_str, &top_n,
37	&index_dir)) {
38	return NULL;	×
39	}
40
41	StatisticsQueryType qt;
42	if (strcmp(query_type_str, "summary") == 0) {	8✔
43	qt = StatisticsQueryType::SUMMARY;	3✔
44	} else if (strcmp(query_type_str, "categories") == 0) {	5✔
45	qt = StatisticsQueryType::CATEGORIES;	2✔
46	} else if (strcmp(query_type_str, "names") == 0) {	3✔
47	qt = StatisticsQueryType::NAMES;	1✔
48	} else if (strcmp(query_type_str, "pid_tids") == 0) {	2!
49	qt = StatisticsQueryType::PID_TIDS;	×
50	} else if (strcmp(query_type_str, "time_range") == 0) {	2!
51	qt = StatisticsQueryType::TIME_RANGE;	×
52	} else if (strcmp(query_type_str, "duration_stats") == 0) {	2✔
53	qt = StatisticsQueryType::DURATION_STATS;	1✔
54	} else if (strcmp(query_type_str, "top_n_names") == 0) {	1!
55	qt = StatisticsQueryType::TOP_N_NAMES;	1✔
UNCOV 56	} else if (strcmp(query_type_str, "top_n_categories") == 0) {	×
57	qt = StatisticsQueryType::TOP_N_CATEGORIES;	×
58	} else if (strcmp(query_type_str, "detailed") == 0) {	×
59	qt = StatisticsQueryType::DETAILED;	×
60	} else {
61	PyErr_Format(PyExc_ValueError, "unknown query_type: '%s'",	×
62	query_type_str);
63	return NULL;	×
64	}
65
66	std::string file_path_str(file_path);	8!
67	std::string index_dir_str(index_dir);	8!
68	TraceStatistics stats;	8!
69	StatisticsQueryOutput output;	8✔
70	auto qt_copy = qt;	8✔
71	auto top_n_copy = static_cast<std::uint64_t>(top_n);	8✔
72
73	if (!run_blocking([&] {	8!
74	Runtime *rt = resolve_runtime(self);	8!
75
76	StatisticsAggregatorInput agg_input;	8✔
77	agg_input.file_path = file_path_str;	8!
78	agg_input.index_dir = index_dir_str;	8!
79	agg_input.index_path = dftracer::utils::utilities::composites::dft::
80	internal::determine_index_path(file_path_str, index_dir_str);	8!
81
82	rt->submit(run_coro_scope(rt->executor(),	16!
83	[file_path_str, index_dir_str](	8!
84	CoroScope &scope) -> CoroTask<void> {
85	co_await indexing::ensure_index_fresh(
86	&scope, "", file_path_str,
87	index_dir_str, false);
88	}),	16!
89	"stats-ensure-fresh")
90	.get();	8!
91
92	auto *stats_p = &stats;	16✔
93	auto agg_task = [stats_p, agg_input]() -> CoroTask<void> {	8!
94	StatisticsAggregatorUtility util;
95	*stats_p = co_await util.process(agg_input);
96	};	24!
97	rt->submit(agg_task(), "stats-agg").get();	8!
98
99	StatisticsQueryInput query_input;	8!
100	query_input.stats = std::move(stats);	8✔
101	query_input.query_type = qt_copy;	8✔
102	query_input.top_n = top_n_copy;	8✔
103
104	auto *out_p = &output;	8✔
105	auto query_task = [out_p, query_input]() -> CoroTask<void> {	8!
106	StatisticsQueryUtility util;
107	*out_p = co_await util.process(query_input);
108	};	24!
109	rt->submit(query_task(), "stats-query").get();	8!
110	})) {	8✔
111	return NULL;	×
112	}
113
114	PyObject *results_list =
115	PyList_New(static_cast<Py_ssize_t>(output.results.size()));	8!
116	if (!results_list) return NULL;	8!
117
118	for (std::size_t i = 0; i < output.results.size(); ++i) {	31✔
119	PyObject *tup = PyTuple_New(2);	23!
120	if (!tup) {	23!
121	Py_DECREF(results_list);
122	return NULL;	×
123	}
124	PyTuple_SET_ITEM(tup, 0,	23!
125	PyUnicode_FromString(output.results[i].first.c_str()));
126	PyTuple_SET_ITEM(tup, 1,	23!
127	PyLong_FromUnsignedLongLong(output.results[i].second));
128	PyList_SET_ITEM(results_list, static_cast<Py_ssize_t>(i), tup);	23✔
129	}
130
131	PyObject *d = PyDict_New();	8!
132	if (!d) {	8!
133	Py_DECREF(results_list);
134	return NULL;	×
135	}
136
137	int rc = 0;	8✔
138	rc \|= dict_set_str(d, "query_type", output.query_type_name.c_str());	8!
139	rc \|= dict_set_u64(d, "total_events", output.total_events);	8!
140	rc \|= dict_set_u64(d, "min_timestamp_us", output.min_timestamp_us);	8!
141	rc \|= dict_set_u64(d, "max_timestamp_us", output.max_timestamp_us);	8!
142	rc \|= dict_set_f64(d, "time_span_seconds", output.time_span_seconds);	8!
143	rc \|= dict_set_u64(d, "duration_count", output.duration_count);	8!
144	rc \|= dict_set_f64(d, "duration_mean_us", output.duration_mean_us);	8!
145	rc \|= dict_set_f64(d, "duration_stddev_us", output.duration_stddev_us);	8!
146	rc \|= dict_set_u64(d, "duration_min_us", output.duration_min_us);	8!
147	rc \|= dict_set_u64(d, "duration_max_us", output.duration_max_us);	8!
148	// Steals (and always releases) the results_list reference.
149	rc \|= dict_set_steal(d, "results", results_list);	8!
150
151	if (rc != 0) {	8!
152	Py_DECREF(d);
153	return NULL;	×
154	}
155
156	return d;	8✔
157	}	8✔
158
159	static PyObject StatisticsQuery_call(PyObject self, PyObject *args,	1✔
160	PyObject *kwds) {
161	return StatisticsQuery_query((StatisticsQueryObject *)self, args, kwds);	1✔
162	}
163
164	static PyMethodDef StatisticsQuery_methods[] = {
165	{"process", (PyCFunction)StatisticsQuery_query,
166	METH_VARARGS \| METH_KEYWORDS,
167	"process(file_path, query_type='summary', top_n=10, index_dir='')\n"
168	"--\n"
169	"\n"
170	"Query statistics from an indexed trace file.\n"
171	"\n"
172	"Args:\n"
173	" file_path (str): Path to the trace file.\n"
174	" query_type (str): Query type (default 'summary'). One of\n"
175	" 'summary', 'categories', 'names', 'pid_tids',\n"
176	" 'time_range', 'duration_stats', 'top_n_names',\n"
177	" 'top_n_categories', 'detailed'.\n"
178	" top_n (int): Top results for ranked queries (default 10).\n"
179	" index_dir (str): Directory for .dftindex stores (default '').\n"
180	"\n"
181	"Returns:\n"
182	" dict: Query results.\n"},
183	{NULL}};
184
185	PyTypeObject StatisticsQueryUtilityType = {
186	PyVarObject_HEAD_INIT(
187	NULL, 0) "dftracer_utils_ext.StatisticsQueryUtility", /* tp_name */
188	sizeof(StatisticsQueryObject), /* tp_basicsize */
189	0, /* tp_itemsize */
190	(destructor)StatisticsQuery_dealloc, /* tp_dealloc */
191	0, /* tp_vectorcall_offset */
192	0, /* tp_getattr */
193	0, /* tp_setattr */
194	0, /* tp_as_async */
195	0, /* tp_repr */
196	0, /* tp_as_number */
197	0, /* tp_as_sequence */
198	0, /* tp_as_mapping */
199	0, /* tp_hash */
200	StatisticsQuery_call, /* tp_call */
201	0, /* tp_str */
202	0, /* tp_getattro */
203	0, /* tp_setattro */
204	0, /* tp_as_buffer */
205	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
206	"StatisticsQueryUtility(runtime: Runtime \| None = None)\n"
207	"--\n\n"
208	"Query pre-computed statistics from an indexed trace file.\n\n"
209	"Args:\n"
210	" runtime (Runtime or None): Runtime for thread pool control.\n",
211	0, /* tp_traverse */
212	0, /* tp_clear */
213	0, /* tp_richcompare */
214	0, /* tp_weaklistoffset */
215	0, /* tp_iter */
216	0, /* tp_iternext */
217	StatisticsQuery_methods, /* tp_methods */
218	0, /* tp_members */
219	0, /* tp_getset */
220	0, /* tp_base */
221	0, /* tp_dict */
222	0, /* tp_descr_get */
223	0, /* tp_descr_set */
224	0, /* tp_dictoffset */
225	(initproc)StatisticsQuery_init, /* tp_init */
226	0, /* tp_alloc */
227	StatisticsQuery_new, /* tp_new */
228	};
229
230	int init_statistics_query(PyObject *m) {	1✔
231	if (register_type(m, &StatisticsQueryUtilityType,	1✔
232	"StatisticsQueryUtility") < 0)	1!
233	return -1;	×
234
235	return 0;	1✔
236	}

llnl / dftracer-utils / 30067416336

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous