23529483807

Committed 25 Mar 2026 07:17AM UTC coverage: 48.515% (-1.6%) from 50.098%

Build # 23529483807

Build Type

Pull #57

github

Committed by

web-flow

Commit Message

Merge 5b1e117ad into 38f9f3616

Pull Request Pull Request #57: feat(comparator): add pairwise traces comparator

Coverage Stats

18829 of 49412 branches covered (38.11%)

Branch coverage included in aggregate %.

1584 of 1933 new or added lines in 14 files covered. (81.95%)

3552 existing lines in 135 files now uncovered.

18474 of 27477 relevant lines covered (67.23%)

241072.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.66

/src/dftracer/utils/python/utilities/statistics_aggregator.cpp

#define PY_SSIZE_T_CLEAN
#include <dftracer/utils/core/coro/task.h>
#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/statistics_aggregator.h>
#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
#include <dftracer/utils/utilities/composites/dft/statistics/trace_statistics.h>

#include <string>

using dftracer::utils::Runtime;
using dftracer::utils::coro::CoroTask;
using namespace dftracer::utils::utilities::composites::dft::statistics;

static Runtime *get_runtime(StatisticsAggregatorObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void StatisticsAggregator_dealloc(StatisticsAggregatorObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *StatisticsAggregator_new(PyTypeObject *type, PyObject *args,
                                          PyObject *kwds) {
    StatisticsAggregatorObject *self =
        (StatisticsAggregatorObject *)type->tp_alloc(type, 0);
    if (self) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int StatisticsAggregator_init(StatisticsAggregatorObject *self,
                                     PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *StatisticsAggregator_compute(StatisticsAggregatorObject *self,
                                              PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"file_path", "index_dir", NULL};
    const char *file_path;
    const char *index_dir = "";
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", (char **)kwlist,
                                     &file_path, &index_dir))
        return NULL;

    std::string file_path_str(file_path);
    std::string index_dir_str(index_dir);
    std::string error_msg;
    TraceStatistics stats;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);

        StatisticsAggregatorInput input;
        input.file_path = file_path_str;
        input.index_dir = index_dir_str;
        input.idx_path = file_path_str + ".idx";

        auto *stats_p = &stats;
        auto input_copy = input;
        auto task = [stats_p, input_copy]() -> CoroTask<void> {
            StatisticsAggregatorUtility util;
            *stats_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "stats-aggregator").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    PyObject *d = PyDict_New();
    if (!d) return NULL;

#define SET_STR(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyUnicode_FromString(v);          \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_ULL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromUnsignedLongLong(v);   \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_SZT(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromSize_t(v);             \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_DBL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyFloat_FromDouble(v);            \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_BOOL(k, v)                                   \
    do {                                                 \
        PyObject *_v = PyBool_FromLong(v ? 1 : 0);       \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

    SET_STR("file_path", stats.file_path.c_str());
    SET_ULL("total_events", stats.total_events());
    SET_ULL("num_chunks", stats.num_chunks);
    SET_BOOL("success", stats.success);
    SET_STR("error_message", stats.error_message.c_str());
    SET_DBL("time_span_seconds", stats.time_span_seconds());
    SET_DBL("duration_mean_us", stats.duration_mean_us());
    SET_DBL("duration_stddev_us", stats.duration_stddev_us());
    SET_SZT("num_categories", stats.num_categories());
    SET_SZT("num_unique_names", stats.num_unique_names());
    SET_SZT("num_pid_tids", stats.num_pid_tids());
    SET_ULL("min_timestamp_us", stats.merged.min_timestamp_us);
    SET_ULL("max_timestamp_us", stats.merged.max_timestamp_us);

#undef SET_STR
#undef SET_ULL
#undef SET_SZT
#undef SET_DBL
#undef SET_BOOL

    return d;
}

static PyObject *StatisticsAggregator_call(PyObject *self, PyObject *args,
                                           PyObject *kwds) {
    return StatisticsAggregator_compute((StatisticsAggregatorObject *)self,
                                        args, kwds);
}

static PyMethodDef StatisticsAggregator_methods[] = {
    {"process", (PyCFunction)StatisticsAggregator_compute,
     METH_VARARGS | METH_KEYWORDS,
     "process(file_path, index_dir='')\n"
     "--\n"
     "\n"
     "Compute aggregated statistics from a trace file.\n"
     "\n"
     "Args:\n"
     "    file_path (str): Path to the trace file.\n"
     "    index_dir (str): Directory for index sidecars (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Aggregated statistics.\n"},
    {NULL}};

PyTypeObject StatisticsAggregatorType = {
    PyVarObject_HEAD_INIT(
        NULL, 0) "dftracer_utils_ext.StatisticsAggregatorUtility", /* tp_name */
    sizeof(StatisticsAggregatorObject),       /* tp_basicsize */
    0,                                        /* tp_itemsize */
    (destructor)StatisticsAggregator_dealloc, /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    StatisticsAggregator_call,                /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "StatisticsAggregatorUtility(runtime: Runtime | None = None)\n"
    "--\n\n"
    "Aggregate statistics from an indexed trace file.\n\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n",
    0,                                   /* tp_traverse */
    0,                                   /* tp_clear */
    0,                                   /* tp_richcompare */
    0,                                   /* tp_weaklistoffset */
    0,                                   /* tp_iter */
    0,                                   /* tp_iternext */
    StatisticsAggregator_methods,        /* tp_methods */
    0,                                   /* tp_members */
    0,                                   /* tp_getset */
    0,                                   /* tp_base */
    0,                                   /* tp_dict */
    0,                                   /* tp_descr_get */
    0,                                   /* tp_descr_set */
    0,                                   /* tp_dictoffset */
    (initproc)StatisticsAggregator_init, /* tp_init */
    0,                                   /* tp_alloc */
    StatisticsAggregator_new,            /* tp_new */
};

int init_statistics_aggregator(PyObject *m) {
    if (PyType_Ready(&StatisticsAggregatorType) < 0) return -1;

    Py_INCREF(&StatisticsAggregatorType);
    if (PyModule_AddObject(m, "StatisticsAggregatorUtility",
                           (PyObject *)&StatisticsAggregatorType) < 0) {
        Py_DECREF(&StatisticsAggregatorType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/core/coro/task.h>
3	#include <dftracer/utils/core/runtime.h>
4	#include <dftracer/utils/python/runtime.h>
5	#include <dftracer/utils/python/utilities/statistics_aggregator.h>
6	#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
7	#include <dftracer/utils/utilities/composites/dft/statistics/trace_statistics.h>
8
9	#include <string>
10
11	using dftracer::utils::Runtime;
12	using dftracer::utils::coro::CoroTask;
13	using namespace dftracer::utils::utilities::composites::dft::statistics;
14
15	static Runtime get_runtime(StatisticsAggregatorObject self) {	4✔
16	if (self->runtime_obj)	4!
17	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
18	return get_default_runtime();	4✔
19	}	4✔
20
21	static void StatisticsAggregator_dealloc(StatisticsAggregatorObject *self) {	4✔
22	Py_XDECREF(self->runtime_obj);	4✔
23	Py_TYPE(self)->tp_free((PyObject *)self);	4✔
24	}	4✔
25
26	static PyObject StatisticsAggregator_new(PyTypeObject type, PyObject *args,	4✔
27	PyObject *kwds) {
28	StatisticsAggregatorObject *self =	4✔
29	(StatisticsAggregatorObject *)type->tp_alloc(type, 0);	4✔
30	if (self) {	4!
31	self->runtime_obj = NULL;	4✔
32	}	4✔
33	return (PyObject *)self;	4✔
34	}
35
36	static int StatisticsAggregator_init(StatisticsAggregatorObject *self,	4✔
37	PyObject args, PyObject kwds) {
38	static const char *kwlist[] = {"runtime", NULL};
39	PyObject *runtime_arg = NULL;	4✔
40
41	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	4!
42	&runtime_arg)) {
43	return -1;	×
44	}
45
46	if (runtime_arg && runtime_arg != Py_None) {	4!
47	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
48	Py_INCREF(runtime_arg);	×
49	self->runtime_obj = runtime_arg;	×
UNCOV 50	} else {	×
51	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
52	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
53	self->runtime_obj = native;	×
UNCOV 54	} else {	×
55	Py_XDECREF(native);	×
56	PyErr_SetString(PyExc_TypeError,	×
57	"runtime must be a Runtime instance or None");
58	return -1;	×
59	}
60	}
UNCOV 61	}	×
62
63	return 0;	4✔
64	}	4✔
65
66	static PyObject StatisticsAggregator_compute(StatisticsAggregatorObject self,	4✔
67	PyObject args, PyObject kwds) {
68	static const char *kwlist[] = {"file_path", "index_dir", NULL};
69	const char *file_path;
70	const char *index_dir = "";	4✔
71	if (!PyArg_ParseTupleAndKeywords(args, kwds, "s\|s", (char **)kwlist,	4!
72	&file_path, &index_dir))
73	return NULL;	×
74
75	std::string file_path_str(file_path);	4✔
76	std::string index_dir_str(index_dir);	4!
77	std::string error_msg;	4✔
78	TraceStatistics stats;	4!
79
80	Py_BEGIN_ALLOW_THREADS try {	4!
81	Runtime *rt = get_runtime(self);	4!
82
83	StatisticsAggregatorInput input;	4✔
84	input.file_path = file_path_str;	4!
85	input.index_dir = index_dir_str;	4!
86	input.idx_path = file_path_str + ".idx";	4!
87
88	auto *stats_p = &stats;	4✔
89	auto input_copy = input;	4!
90	auto task = [stats_p, input_copy]() -> CoroTask<void> {	32!
91	StatisticsAggregatorUtility util;	12!
92	*stats_p = co_await util.process(input_copy);	16!
93	};	12!
94	rt->submit(task(), "stats-aggregator").get();	4!
95	} catch (const std::exception &e) {	4!
96	error_msg = e.what();	×
97	}	×
98	Py_END_ALLOW_THREADS	4!
99
100	if (!error_msg.empty()) {	4!
101	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
102	return NULL;	×
103	}
104
105	PyObject *d = PyDict_New();	4!
106	if (!d) return NULL;	4!
107
108	#define SET_STR(k, v) \
109	do { \
110	PyObject *_v = PyUnicode_FromString(v); \
111	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
112	Py_XDECREF(_v); \
113	Py_DECREF(d); \
114	return NULL; \
115	} \
116	Py_DECREF(_v); \
117	} while (0)
118
119	#define SET_ULL(k, v) \
120	do { \
121	PyObject *_v = PyLong_FromUnsignedLongLong(v); \
122	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
123	Py_XDECREF(_v); \
124	Py_DECREF(d); \
125	return NULL; \
126	} \
127	Py_DECREF(_v); \
128	} while (0)
129
130	#define SET_SZT(k, v) \
131	do { \
132	PyObject *_v = PyLong_FromSize_t(v); \
133	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
134	Py_XDECREF(_v); \
135	Py_DECREF(d); \
136	return NULL; \
137	} \
138	Py_DECREF(_v); \
139	} while (0)
140
141	#define SET_DBL(k, v) \
142	do { \
143	PyObject *_v = PyFloat_FromDouble(v); \
144	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
145	Py_XDECREF(_v); \
146	Py_DECREF(d); \
147	return NULL; \
148	} \
149	Py_DECREF(_v); \
150	} while (0)
151
152	#define SET_BOOL(k, v) \
153	do { \
154	PyObject *_v = PyBool_FromLong(v ? 1 : 0); \
155	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
156	Py_XDECREF(_v); \
157	Py_DECREF(d); \
158	return NULL; \
159	} \
160	Py_DECREF(_v); \
161	} while (0)
162
163	SET_STR("file_path", stats.file_path.c_str());	4!
164	SET_ULL("total_events", stats.total_events());	4!
165	SET_ULL("num_chunks", stats.num_chunks);	4!
166	SET_BOOL("success", stats.success);	4!
167	SET_STR("error_message", stats.error_message.c_str());	4!
168	SET_DBL("time_span_seconds", stats.time_span_seconds());	4!
169	SET_DBL("duration_mean_us", stats.duration_mean_us());	4!
170	SET_DBL("duration_stddev_us", stats.duration_stddev_us());	4!
171	SET_SZT("num_categories", stats.num_categories());	4!
172	SET_SZT("num_unique_names", stats.num_unique_names());	4!
173	SET_SZT("num_pid_tids", stats.num_pid_tids());	4!
174	SET_ULL("min_timestamp_us", stats.merged.min_timestamp_us);	4!
175	SET_ULL("max_timestamp_us", stats.merged.max_timestamp_us);	4!
176
177	#undef SET_STR
178	#undef SET_ULL
179	#undef SET_SZT
180	#undef SET_DBL
181	#undef SET_BOOL
182
183	return d;	4✔
184	}	4✔
185
186	static PyObject StatisticsAggregator_call(PyObject self, PyObject *args,	1✔
187	PyObject *kwds) {
188	return StatisticsAggregator_compute((StatisticsAggregatorObject *)self,	2✔
189	args, kwds);	1✔
190	}
191
192	static PyMethodDef StatisticsAggregator_methods[] = {
193	{"process", (PyCFunction)StatisticsAggregator_compute,
194	METH_VARARGS \| METH_KEYWORDS,
195	"process(file_path, index_dir='')\n"
196	"--\n"
197	"\n"
198	"Compute aggregated statistics from a trace file.\n"
199	"\n"
200	"Args:\n"
201	" file_path (str): Path to the trace file.\n"
202	" index_dir (str): Directory for index sidecars (default '').\n"
203	"\n"
204	"Returns:\n"
205	" dict: Aggregated statistics.\n"},
206	{NULL}};
207
208	PyTypeObject StatisticsAggregatorType = {
209	PyVarObject_HEAD_INIT(
210	NULL, 0) "dftracer_utils_ext.StatisticsAggregatorUtility", /* tp_name */
211	sizeof(StatisticsAggregatorObject), /* tp_basicsize */
212	0, /* tp_itemsize */
213	(destructor)StatisticsAggregator_dealloc, /* tp_dealloc */
214	0, /* tp_vectorcall_offset */
215	0, /* tp_getattr */
216	0, /* tp_setattr */
217	0, /* tp_as_async */
218	0, /* tp_repr */
219	0, /* tp_as_number */
220	0, /* tp_as_sequence */
221	0, /* tp_as_mapping */
222	0, /* tp_hash */
223	StatisticsAggregator_call, /* tp_call */
224	0, /* tp_str */
225	0, /* tp_getattro */
226	0, /* tp_setattro */
227	0, /* tp_as_buffer */
228	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
229	"StatisticsAggregatorUtility(runtime: Runtime \| None = None)\n"
230	"--\n\n"
231	"Aggregate statistics from an indexed trace file.\n\n"
232	"Args:\n"
233	" runtime (Runtime or None): Runtime for thread pool control.\n",
234	0, /* tp_traverse */
235	0, /* tp_clear */
236	0, /* tp_richcompare */
237	0, /* tp_weaklistoffset */
238	0, /* tp_iter */
239	0, /* tp_iternext */
240	StatisticsAggregator_methods, /* tp_methods */
241	0, /* tp_members */
242	0, /* tp_getset */
243	0, /* tp_base */
244	0, /* tp_dict */
245	0, /* tp_descr_get */
246	0, /* tp_descr_set */
247	0, /* tp_dictoffset */
248	(initproc)StatisticsAggregator_init, /* tp_init */
249	0, /* tp_alloc */
250	StatisticsAggregator_new, /* tp_new */
251	};
252
253	int init_statistics_aggregator(PyObject *m) {	1✔
254	if (PyType_Ready(&StatisticsAggregatorType) < 0) return -1;	1!
255
256	Py_INCREF(&StatisticsAggregatorType);	1✔
257	if (PyModule_AddObject(m, "StatisticsAggregatorUtility",	2!
258	(PyObject *)&StatisticsAggregatorType) < 0) {	1✔
UNCOV 259	Py_DECREF(&StatisticsAggregatorType);	×
UNCOV 260	Py_DECREF(m);	×
261	return -1;	×
262	}
263
264	return 0;	1✔
265	}	1✔

llnl / dftracer-utils / 23529483807

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous