27052412546

Committed 06 Jun 2026 04:20AM UTC coverage: 50.862% (+1.0%) from 49.905%

Build # 27052412546

Build Type

Pull #73

github

Committed by

web-flow

Commit Message

Merge 734572730 into 88a3c8457

Pull Request Pull Request #73: add portable dependencies wheel support

Coverage Stats

31801 of 79859 branches covered (39.82%)

Branch coverage included in aggregate %.

32491 of 46545 relevant lines covered (69.81%)

9947.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.9

/src/dftracer/utils/python/utilities/statistics_aggregator.cpp

#define PY_SSIZE_T_CLEAN
#include <dftracer/utils/core/coro/task.h>
#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/statistics_aggregator.h>
#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
#include <dftracer/utils/utilities/composites/dft/statistics/trace_statistics.h>

#include <string>

using dftracer::utils::Runtime;
using dftracer::utils::coro::CoroTask;
using namespace dftracer::utils::utilities::composites::dft::statistics;

static Runtime *get_runtime(StatisticsAggregatorObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void StatisticsAggregator_dealloc(StatisticsAggregatorObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *StatisticsAggregator_new(PyTypeObject *type, PyObject *args,
                                          PyObject *kwds) {
    StatisticsAggregatorObject *self =
        (StatisticsAggregatorObject *)type->tp_alloc(type, 0);
    if (self) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int StatisticsAggregator_init(StatisticsAggregatorObject *self,
                                     PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *StatisticsAggregator_compute(StatisticsAggregatorObject *self,
                                              PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"file_path", "index_dir", NULL};
    const char *file_path;
    const char *index_dir = "";
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", (char **)kwlist,
                                     &file_path, &index_dir))
        return NULL;

    std::string file_path_str(file_path);
    std::string index_dir_str(index_dir);
    std::string error_msg;
    TraceStatistics stats;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);

        StatisticsAggregatorInput input;
        input.file_path = file_path_str;
        input.index_dir = index_dir_str;
        input.index_path = dftracer::utils::utilities::composites::dft::
            internal::determine_index_path(file_path_str, index_dir_str);

        auto *stats_p = &stats;
        auto input_copy = input;
        auto task = [stats_p, input_copy]() -> CoroTask<void> {
            StatisticsAggregatorUtility util;
            *stats_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "stats-aggregator").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    PyObject *d = PyDict_New();
    if (!d) return NULL;

#define SET_STR(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyUnicode_FromString(v);          \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_ULL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromUnsignedLongLong(v);   \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_SZT(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromSize_t(v);             \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_DBL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyFloat_FromDouble(v);            \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_BOOL(k, v)                                   \
    do {                                                 \
        PyObject *_v = PyBool_FromLong(v ? 1 : 0);       \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

    SET_STR("file_path", stats.file_path.c_str());
    SET_ULL("total_events", stats.total_events());
    SET_ULL("num_chunks", stats.num_chunks);
    SET_BOOL("success", stats.success);
    SET_STR("error_message", stats.error_message.c_str());
    SET_DBL("time_span_seconds", stats.time_span_seconds());
    SET_DBL("duration_mean_us", stats.duration_mean_us());
    SET_DBL("duration_stddev_us", stats.duration_stddev_us());
    SET_SZT("num_categories", stats.num_categories());
    SET_SZT("num_unique_names", stats.num_unique_names());
    SET_SZT("num_pid_tids", stats.num_pid_tids());
    SET_ULL("min_timestamp_us", stats.merged.min_timestamp_us);
    SET_ULL("max_timestamp_us", stats.merged.max_timestamp_us);

#undef SET_STR
#undef SET_ULL
#undef SET_SZT
#undef SET_DBL
#undef SET_BOOL

    return d;
}

static PyObject *StatisticsAggregator_call(PyObject *self, PyObject *args,
                                           PyObject *kwds) {
    return StatisticsAggregator_compute((StatisticsAggregatorObject *)self,
                                        args, kwds);
}

static PyMethodDef StatisticsAggregator_methods[] = {
    {"process", (PyCFunction)StatisticsAggregator_compute,
     METH_VARARGS | METH_KEYWORDS,
     "process(file_path, index_dir='')\n"
     "--\n"
     "\n"
     "Compute aggregated statistics from a trace file.\n"
     "\n"
     "Args:\n"
     "    file_path (str): Path to the trace file.\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Aggregated statistics.\n"},
    {NULL}};

PyTypeObject StatisticsAggregatorType = {
    PyVarObject_HEAD_INIT(
        NULL, 0) "dftracer_utils_ext.StatisticsAggregatorUtility", /* tp_name */
    sizeof(StatisticsAggregatorObject),       /* tp_basicsize */
    0,                                        /* tp_itemsize */
    (destructor)StatisticsAggregator_dealloc, /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    StatisticsAggregator_call,                /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "StatisticsAggregatorUtility(runtime: Runtime | None = None)\n"
    "--\n\n"
    "Aggregate statistics from an indexed trace file.\n\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n",
    0,                                   /* tp_traverse */
    0,                                   /* tp_clear */
    0,                                   /* tp_richcompare */
    0,                                   /* tp_weaklistoffset */
    0,                                   /* tp_iter */
    0,                                   /* tp_iternext */
    StatisticsAggregator_methods,        /* tp_methods */
    0,                                   /* tp_members */
    0,                                   /* tp_getset */
    0,                                   /* tp_base */
    0,                                   /* tp_dict */
    0,                                   /* tp_descr_get */
    0,                                   /* tp_descr_set */
    0,                                   /* tp_dictoffset */
    (initproc)StatisticsAggregator_init, /* tp_init */
    0,                                   /* tp_alloc */
    StatisticsAggregator_new,            /* tp_new */
};

int init_statistics_aggregator(PyObject *m) {
    if (PyType_Ready(&StatisticsAggregatorType) < 0) return -1;

    Py_INCREF(&StatisticsAggregatorType);
    if (PyModule_AddObject(m, "StatisticsAggregatorUtility",
                           (PyObject *)&StatisticsAggregatorType) < 0) {
        Py_DECREF(&StatisticsAggregatorType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/core/coro/task.h>
3	#include <dftracer/utils/core/runtime.h>
4	#include <dftracer/utils/python/runtime.h>
5	#include <dftracer/utils/python/utilities/statistics_aggregator.h>
6	#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
7	#include <dftracer/utils/utilities/composites/dft/statistics/statistics_aggregator_utility.h>
8	#include <dftracer/utils/utilities/composites/dft/statistics/trace_statistics.h>
9
10	#include <string>
11
12	using dftracer::utils::Runtime;
13	using dftracer::utils::coro::CoroTask;
14	using namespace dftracer::utils::utilities::composites::dft::statistics;
15
16	static Runtime get_runtime(StatisticsAggregatorObject self) {	7✔
17	if (self->runtime_obj)	7!
18	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
19	return get_default_runtime();	7✔
20	}	7✔
21
22	static void StatisticsAggregator_dealloc(StatisticsAggregatorObject *self) {	7✔
23	Py_XDECREF(self->runtime_obj);	7✔
24	Py_TYPE(self)->tp_free((PyObject *)self);	7✔
25	}	7✔
26
27	static PyObject StatisticsAggregator_new(PyTypeObject type, PyObject *args,	7✔
28	PyObject *kwds) {
29	StatisticsAggregatorObject *self =	7✔
30	(StatisticsAggregatorObject *)type->tp_alloc(type, 0);	7✔
31	if (self) {	7!
32	self->runtime_obj = NULL;	7✔
33	}	7✔
34	return (PyObject *)self;	7✔
35	}
36
37	static int StatisticsAggregator_init(StatisticsAggregatorObject *self,	7✔
38	PyObject args, PyObject kwds) {
39	static const char *kwlist[] = {"runtime", NULL};
40	PyObject *runtime_arg = NULL;	7✔
41
42	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	7!
43	&runtime_arg)) {
44	return -1;	×
45	}
46
47	if (runtime_arg && runtime_arg != Py_None) {	7!
48	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
49	Py_INCREF(runtime_arg);	×
50	self->runtime_obj = runtime_arg;	×
51	} else {	×
52	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
53	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
54	self->runtime_obj = native;	×
55	} else {	×
56	Py_XDECREF(native);	×
57	PyErr_SetString(PyExc_TypeError,	×
58	"runtime must be a Runtime instance or None");
59	return -1;	×
60	}
61	}
62	}	×
63
64	return 0;	7✔
65	}	7✔
66
67	static PyObject StatisticsAggregator_compute(StatisticsAggregatorObject self,	7✔
68	PyObject args, PyObject kwds) {
69	static const char *kwlist[] = {"file_path", "index_dir", NULL};
70	const char *file_path;
71	const char *index_dir = "";	7✔
72	if (!PyArg_ParseTupleAndKeywords(args, kwds, "s\|s", (char **)kwlist,	7!
73	&file_path, &index_dir))
74	return NULL;	×
75
76	std::string file_path_str(file_path);	7✔
77	std::string index_dir_str(index_dir);	7!
78	std::string error_msg;	7✔
79	TraceStatistics stats;	7!
80
81	Py_BEGIN_ALLOW_THREADS try {	7!
82	Runtime *rt = get_runtime(self);	7!
83
84	StatisticsAggregatorInput input;	7✔
85	input.file_path = file_path_str;	7!
86	input.index_dir = index_dir_str;	7!
87	input.index_path = dftracer::utils::utilities::composites::dft::	7!
88	internal::determine_index_path(file_path_str, index_dir_str);
89
90	auto *stats_p = &stats;	7✔
91	auto input_copy = input;	7!
92	auto task = [stats_p, input_copy]() -> CoroTask<void> {	56!
93	StatisticsAggregatorUtility util;	21!
94	*stats_p = co_await util.process(input_copy);	28!
95	};	21!
96	rt->submit(task(), "stats-aggregator").get();	7!
97	} catch (const std::exception &e) {	7!
98	error_msg = e.what();	×
99	}	×
100	Py_END_ALLOW_THREADS	7!
101
102	if (!error_msg.empty()) {	7!
103	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
104	return NULL;	×
105	}
106
107	PyObject *d = PyDict_New();	7!
108	if (!d) return NULL;	7!
109
110	#define SET_STR(k, v) \
111	do { \
112	PyObject *_v = PyUnicode_FromString(v); \
113	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
114	Py_XDECREF(_v); \
115	Py_DECREF(d); \
116	return NULL; \
117	} \
118	Py_DECREF(_v); \
119	} while (0)
120
121	#define SET_ULL(k, v) \
122	do { \
123	PyObject *_v = PyLong_FromUnsignedLongLong(v); \
124	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
125	Py_XDECREF(_v); \
126	Py_DECREF(d); \
127	return NULL; \
128	} \
129	Py_DECREF(_v); \
130	} while (0)
131
132	#define SET_SZT(k, v) \
133	do { \
134	PyObject *_v = PyLong_FromSize_t(v); \
135	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
136	Py_XDECREF(_v); \
137	Py_DECREF(d); \
138	return NULL; \
139	} \
140	Py_DECREF(_v); \
141	} while (0)
142
143	#define SET_DBL(k, v) \
144	do { \
145	PyObject *_v = PyFloat_FromDouble(v); \
146	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
147	Py_XDECREF(_v); \
148	Py_DECREF(d); \
149	return NULL; \
150	} \
151	Py_DECREF(_v); \
152	} while (0)
153
154	#define SET_BOOL(k, v) \
155	do { \
156	PyObject *_v = PyBool_FromLong(v ? 1 : 0); \
157	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
158	Py_XDECREF(_v); \
159	Py_DECREF(d); \
160	return NULL; \
161	} \
162	Py_DECREF(_v); \
163	} while (0)
164
165	SET_STR("file_path", stats.file_path.c_str());	7!
166	SET_ULL("total_events", stats.total_events());	7!
167	SET_ULL("num_chunks", stats.num_chunks);	7!
168	SET_BOOL("success", stats.success);	7!
169	SET_STR("error_message", stats.error_message.c_str());	7!
170	SET_DBL("time_span_seconds", stats.time_span_seconds());	7!
171	SET_DBL("duration_mean_us", stats.duration_mean_us());	7!
172	SET_DBL("duration_stddev_us", stats.duration_stddev_us());	7!
173	SET_SZT("num_categories", stats.num_categories());	7!
174	SET_SZT("num_unique_names", stats.num_unique_names());	7!
175	SET_SZT("num_pid_tids", stats.num_pid_tids());	7!
176	SET_ULL("min_timestamp_us", stats.merged.min_timestamp_us);	7!
177	SET_ULL("max_timestamp_us", stats.merged.max_timestamp_us);	7!
178
179	#undef SET_STR
180	#undef SET_ULL
181	#undef SET_SZT
182	#undef SET_DBL
183	#undef SET_BOOL
184
185	return d;	7✔
186	}	7✔
187
188	static PyObject StatisticsAggregator_call(PyObject self, PyObject *args,	1✔
189	PyObject *kwds) {
190	return StatisticsAggregator_compute((StatisticsAggregatorObject *)self,	2✔
191	args, kwds);	1✔
192	}
193
194	static PyMethodDef StatisticsAggregator_methods[] = {
195	{"process", (PyCFunction)StatisticsAggregator_compute,
196	METH_VARARGS \| METH_KEYWORDS,
197	"process(file_path, index_dir='')\n"
198	"--\n"
199	"\n"
200	"Compute aggregated statistics from a trace file.\n"
201	"\n"
202	"Args:\n"
203	" file_path (str): Path to the trace file.\n"
204	" index_dir (str): Directory for .dftindex stores (default '').\n"
205	"\n"
206	"Returns:\n"
207	" dict: Aggregated statistics.\n"},
208	{NULL}};
209
210	PyTypeObject StatisticsAggregatorType = {
211	PyVarObject_HEAD_INIT(
212	NULL, 0) "dftracer_utils_ext.StatisticsAggregatorUtility", /* tp_name */
213	sizeof(StatisticsAggregatorObject), /* tp_basicsize */
214	0, /* tp_itemsize */
215	(destructor)StatisticsAggregator_dealloc, /* tp_dealloc */
216	0, /* tp_vectorcall_offset */
217	0, /* tp_getattr */
218	0, /* tp_setattr */
219	0, /* tp_as_async */
220	0, /* tp_repr */
221	0, /* tp_as_number */
222	0, /* tp_as_sequence */
223	0, /* tp_as_mapping */
224	0, /* tp_hash */
225	StatisticsAggregator_call, /* tp_call */
226	0, /* tp_str */
227	0, /* tp_getattro */
228	0, /* tp_setattro */
229	0, /* tp_as_buffer */
230	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
231	"StatisticsAggregatorUtility(runtime: Runtime \| None = None)\n"
232	"--\n\n"
233	"Aggregate statistics from an indexed trace file.\n\n"
234	"Args:\n"
235	" runtime (Runtime or None): Runtime for thread pool control.\n",
236	0, /* tp_traverse */
237	0, /* tp_clear */
238	0, /* tp_richcompare */
239	0, /* tp_weaklistoffset */
240	0, /* tp_iter */
241	0, /* tp_iternext */
242	StatisticsAggregator_methods, /* tp_methods */
243	0, /* tp_members */
244	0, /* tp_getset */
245	0, /* tp_base */
246	0, /* tp_dict */
247	0, /* tp_descr_get */
248	0, /* tp_descr_set */
249	0, /* tp_dictoffset */
250	(initproc)StatisticsAggregator_init, /* tp_init */
251	0, /* tp_alloc */
252	StatisticsAggregator_new, /* tp_new */
253	};
254
255	int init_statistics_aggregator(PyObject *m) {	1✔
256	if (PyType_Ready(&StatisticsAggregatorType) < 0) return -1;	1!
257
258	Py_INCREF(&StatisticsAggregatorType);	1✔
259	if (PyModule_AddObject(m, "StatisticsAggregatorUtility",	2!
260	(PyObject *)&StatisticsAggregatorType) < 0) {	1✔
261	Py_DECREF(&StatisticsAggregatorType);	×
262	Py_DECREF(m);	×
263	return -1;	×
264	}
265
266	return 0;	1✔
267	}	1✔

llnl / dftracer-utils / 27052412546

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous