23529483807

Committed 25 Mar 2026 07:17AM UTC coverage: 48.515% (-1.6%) from 50.098%

Build # 23529483807

Build Type

Pull #57

github

Committed by

web-flow

Commit Message

Merge 5b1e117ad into 38f9f3616

Pull Request Pull Request #57: feat(comparator): add pairwise traces comparator

Coverage Stats

18829 of 49412 branches covered (38.11%)

Branch coverage included in aggregate %.

1584 of 1933 new or added lines in 14 files covered. (81.95%)

3552 existing lines in 135 files now uncovered.

18474 of 27477 relevant lines covered (67.23%)

241072.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.47

/src/dftracer/utils/python/utilities/metadata_collector.cpp

#define PY_SSIZE_T_CLEAN
#include <dftracer/utils/core/common/archive_format.h>
#include <dftracer/utils/core/coro/task.h>
#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/metadata_collector.h>
#include <dftracer/utils/utilities/composites/dft/metadata_collector_utility.h>

#include <string>

using dftracer::utils::get_format_name;
using dftracer::utils::Runtime;
using dftracer::utils::coro::CoroTask;
using namespace dftracer::utils::utilities::composites::dft;

static Runtime *get_runtime(MetadataCollectorObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void MetadataCollector_dealloc(MetadataCollectorObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *MetadataCollector_new(PyTypeObject *type, PyObject *args,
                                       PyObject *kwds) {
    MetadataCollectorObject *self =
        (MetadataCollectorObject *)type->tp_alloc(type, 0);
    if (self) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int MetadataCollector_init(MetadataCollectorObject *self, PyObject *args,
                                  PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *MetadataCollector_collect(MetadataCollectorObject *self,
                                           PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"file_path", "index_dir", NULL};
    const char *file_path;
    const char *index_dir = "";
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", (char **)kwlist,
                                     &file_path, &index_dir))
        return NULL;

    std::string file_path_str(file_path);
    std::string error_msg;
    MetadataCollectorUtilityOutput output;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);

        MetadataCollectorUtilityInput input;
        input.file_path = file_path_str;
        input.idx_path = file_path_str + ".idx";

        auto *out_p = &output;
        auto input_copy = input;
        auto task = [out_p, input_copy]() -> CoroTask<void> {
            MetadataCollectorUtility util;
            *out_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "metadata-collector").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    PyObject *d = PyDict_New();
    if (!d) return NULL;

#define SET_STR(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyUnicode_FromString(v);          \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_DBL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyFloat_FromDouble(v);            \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_SZT(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromSize_t(v);             \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_ULL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromUnsignedLongLong(v);   \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_BOOL(k, v)                                   \
    do {                                                 \
        PyObject *_v = PyBool_FromLong(v ? 1 : 0);       \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

    SET_STR("file_path", output.file_path.c_str());
    SET_STR("idx_path", output.idx_path.c_str());
    SET_DBL("size_mb", output.size_mb);
    SET_SZT("start_line", output.start_line);
    SET_SZT("end_line", output.end_line);
    SET_SZT("valid_events", output.valid_events);
    SET_DBL("size_per_line", output.size_per_line);
    SET_BOOL("success", output.success);
    SET_BOOL("has_index", output.has_index);
    SET_BOOL("index_valid", output.index_valid);
    SET_ULL("compressed_size", output.compressed_size);
    SET_ULL("uncompressed_size", output.uncompressed_size);
    SET_ULL("num_lines", output.num_lines);
    SET_ULL("checkpoint_size", output.checkpoint_size);
    SET_SZT("num_checkpoints", output.num_checkpoints);
    SET_STR("format", get_format_name(output.format));
    SET_STR("error_message", output.error_message.c_str());

#undef SET_STR
#undef SET_DBL
#undef SET_SZT
#undef SET_ULL
#undef SET_BOOL

    return d;
}

static PyObject *MetadataCollector_call(PyObject *self, PyObject *args,
                                        PyObject *kwds) {
    return MetadataCollector_collect((MetadataCollectorObject *)self, args,
                                     kwds);
}

static PyMethodDef MetadataCollector_methods[] = {
    {"process", (PyCFunction)MetadataCollector_collect,
     METH_VARARGS | METH_KEYWORDS,
     "Collect metadata from a trace file.\n"
     "\n"
     "Args:\n"
     "    file_path (str): Path to the trace file.\n"
     "    index_dir (str): Directory for index sidecars.\n"},
    {NULL}};

PyTypeObject MetadataCollectorType = {
    PyVarObject_HEAD_INIT(
        NULL, 0) "dftracer_utils_ext.MetadataCollectorUtility", /* tp_name */
    sizeof(MetadataCollectorObject),          /* tp_basicsize */
    0,                                        /* tp_itemsize */
    (destructor)MetadataCollector_dealloc,    /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    MetadataCollector_call,                   /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "MetadataCollectorUtility(runtime: Runtime | None = None)\n"
    "--\n\n"
    "Collect metadata from a DFTracer trace file.\n\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "\n"
    "process(file_path, index_dir='') -> dict\n"
    "    file_path (str): Path to the trace file.\n"
    "    index_dir (str): Directory for index sidecar files.\n",
    0,                                /* tp_traverse */
    0,                                /* tp_clear */
    0,                                /* tp_richcompare */
    0,                                /* tp_weaklistoffset */
    0,                                /* tp_iter */
    0,                                /* tp_iternext */
    MetadataCollector_methods,        /* tp_methods */
    0,                                /* tp_members */
    0,                                /* tp_getset */
    0,                                /* tp_base */
    0,                                /* tp_dict */
    0,                                /* tp_descr_get */
    0,                                /* tp_descr_set */
    0,                                /* tp_dictoffset */
    (initproc)MetadataCollector_init, /* tp_init */
    0,                                /* tp_alloc */
    MetadataCollector_new,            /* tp_new */
};

int init_metadata_collector(PyObject *m) {
    if (PyType_Ready(&MetadataCollectorType) < 0) return -1;

    Py_INCREF(&MetadataCollectorType);
    if (PyModule_AddObject(m, "MetadataCollectorUtility",
                           (PyObject *)&MetadataCollectorType) < 0) {
        Py_DECREF(&MetadataCollectorType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/core/common/archive_format.h>
3	#include <dftracer/utils/core/coro/task.h>
4	#include <dftracer/utils/core/runtime.h>
5	#include <dftracer/utils/python/runtime.h>
6	#include <dftracer/utils/python/utilities/metadata_collector.h>
7	#include <dftracer/utils/utilities/composites/dft/metadata_collector_utility.h>
8
9	#include <string>
10
11	using dftracer::utils::get_format_name;
12	using dftracer::utils::Runtime;
13	using dftracer::utils::coro::CoroTask;
14	using namespace dftracer::utils::utilities::composites::dft;
15
16	static Runtime get_runtime(MetadataCollectorObject self) {	4✔
17	if (self->runtime_obj)	4!
18	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
19	return get_default_runtime();	4✔
20	}	4✔
21
22	static void MetadataCollector_dealloc(MetadataCollectorObject *self) {	4✔
23	Py_XDECREF(self->runtime_obj);	4✔
24	Py_TYPE(self)->tp_free((PyObject *)self);	4✔
25	}	4✔
26
27	static PyObject MetadataCollector_new(PyTypeObject type, PyObject *args,	4✔
28	PyObject *kwds) {
29	MetadataCollectorObject *self =	4✔
30	(MetadataCollectorObject *)type->tp_alloc(type, 0);	4✔
31	if (self) {	4!
32	self->runtime_obj = NULL;	4✔
33	}	4✔
34	return (PyObject *)self;	4✔
35	}
36
37	static int MetadataCollector_init(MetadataCollectorObject self, PyObject args,	4✔
38	PyObject *kwds) {
39	static const char *kwlist[] = {"runtime", NULL};
40	PyObject *runtime_arg = NULL;	4✔
41
42	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	4!
43	&runtime_arg)) {
44	return -1;	×
45	}
46
47	if (runtime_arg && runtime_arg != Py_None) {	4!
48	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
49	Py_INCREF(runtime_arg);	×
50	self->runtime_obj = runtime_arg;	×
UNCOV 51	} else {	×
52	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
53	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
54	self->runtime_obj = native;	×
UNCOV 55	} else {	×
56	Py_XDECREF(native);	×
57	PyErr_SetString(PyExc_TypeError,	×
58	"runtime must be a Runtime instance or None");
59	return -1;	×
60	}
61	}
UNCOV 62	}	×
63
64	return 0;	4✔
65	}	4✔
66
67	static PyObject MetadataCollector_collect(MetadataCollectorObject self,	4✔
68	PyObject args, PyObject kwds) {
69	static const char *kwlist[] = {"file_path", "index_dir", NULL};
70	const char *file_path;
71	const char *index_dir = "";	4✔
72	if (!PyArg_ParseTupleAndKeywords(args, kwds, "s\|s", (char **)kwlist,	4!
73	&file_path, &index_dir))
74	return NULL;	×
75
76	std::string file_path_str(file_path);	4✔
77	std::string error_msg;	4✔
78	MetadataCollectorUtilityOutput output;	4✔
79
80	Py_BEGIN_ALLOW_THREADS try {	4!
81	Runtime *rt = get_runtime(self);	4!
82
83	MetadataCollectorUtilityInput input;	4✔
84	input.file_path = file_path_str;	4!
85	input.idx_path = file_path_str + ".idx";	4!
86
87	auto *out_p = &output;	4✔
88	auto input_copy = input;	4!
89	auto task = [out_p, input_copy]() -> CoroTask<void> {	32!
90	MetadataCollectorUtility util;	12!
91	*out_p = co_await util.process(input_copy);	16!
92	};	12!
93	rt->submit(task(), "metadata-collector").get();	4!
94	} catch (const std::exception &e) {	4!
95	error_msg = e.what();	×
96	}	×
97	Py_END_ALLOW_THREADS	4!
98
99	if (!error_msg.empty()) {	4!
100	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
101	return NULL;	×
102	}
103
104	PyObject *d = PyDict_New();	4!
105	if (!d) return NULL;	4!
106
107	#define SET_STR(k, v) \
108	do { \
109	PyObject *_v = PyUnicode_FromString(v); \
110	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
111	Py_XDECREF(_v); \
112	Py_DECREF(d); \
113	return NULL; \
114	} \
115	Py_DECREF(_v); \
116	} while (0)
117
118	#define SET_DBL(k, v) \
119	do { \
120	PyObject *_v = PyFloat_FromDouble(v); \
121	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
122	Py_XDECREF(_v); \
123	Py_DECREF(d); \
124	return NULL; \
125	} \
126	Py_DECREF(_v); \
127	} while (0)
128
129	#define SET_SZT(k, v) \
130	do { \
131	PyObject *_v = PyLong_FromSize_t(v); \
132	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
133	Py_XDECREF(_v); \
134	Py_DECREF(d); \
135	return NULL; \
136	} \
137	Py_DECREF(_v); \
138	} while (0)
139
140	#define SET_ULL(k, v) \
141	do { \
142	PyObject *_v = PyLong_FromUnsignedLongLong(v); \
143	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
144	Py_XDECREF(_v); \
145	Py_DECREF(d); \
146	return NULL; \
147	} \
148	Py_DECREF(_v); \
149	} while (0)
150
151	#define SET_BOOL(k, v) \
152	do { \
153	PyObject *_v = PyBool_FromLong(v ? 1 : 0); \
154	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
155	Py_XDECREF(_v); \
156	Py_DECREF(d); \
157	return NULL; \
158	} \
159	Py_DECREF(_v); \
160	} while (0)
161
162	SET_STR("file_path", output.file_path.c_str());	4!
163	SET_STR("idx_path", output.idx_path.c_str());	4!
164	SET_DBL("size_mb", output.size_mb);	4!
165	SET_SZT("start_line", output.start_line);	4!
166	SET_SZT("end_line", output.end_line);	4!
167	SET_SZT("valid_events", output.valid_events);	4!
168	SET_DBL("size_per_line", output.size_per_line);	4!
169	SET_BOOL("success", output.success);	4!
170	SET_BOOL("has_index", output.has_index);	4!
171	SET_BOOL("index_valid", output.index_valid);	4!
172	SET_ULL("compressed_size", output.compressed_size);	4!
173	SET_ULL("uncompressed_size", output.uncompressed_size);	4!
174	SET_ULL("num_lines", output.num_lines);	4!
175	SET_ULL("checkpoint_size", output.checkpoint_size);	4!
176	SET_SZT("num_checkpoints", output.num_checkpoints);	4!
177	SET_STR("format", get_format_name(output.format));	4!
178	SET_STR("error_message", output.error_message.c_str());	4!
179
180	#undef SET_STR
181	#undef SET_DBL
182	#undef SET_SZT
183	#undef SET_ULL
184	#undef SET_BOOL
185
186	return d;	4✔
187	}	4✔
188
189	static PyObject MetadataCollector_call(PyObject self, PyObject *args,	1✔
190	PyObject *kwds) {
191	return MetadataCollector_collect((MetadataCollectorObject *)self, args,	2✔
192	kwds);	1✔
193	}
194
195	static PyMethodDef MetadataCollector_methods[] = {
196	{"process", (PyCFunction)MetadataCollector_collect,
197	METH_VARARGS \| METH_KEYWORDS,
198	"Collect metadata from a trace file.\n"
199	"\n"
200	"Args:\n"
201	" file_path (str): Path to the trace file.\n"
202	" index_dir (str): Directory for index sidecars.\n"},
203	{NULL}};
204
205	PyTypeObject MetadataCollectorType = {
206	PyVarObject_HEAD_INIT(
207	NULL, 0) "dftracer_utils_ext.MetadataCollectorUtility", /* tp_name */
208	sizeof(MetadataCollectorObject), /* tp_basicsize */
209	0, /* tp_itemsize */
210	(destructor)MetadataCollector_dealloc, /* tp_dealloc */
211	0, /* tp_vectorcall_offset */
212	0, /* tp_getattr */
213	0, /* tp_setattr */
214	0, /* tp_as_async */
215	0, /* tp_repr */
216	0, /* tp_as_number */
217	0, /* tp_as_sequence */
218	0, /* tp_as_mapping */
219	0, /* tp_hash */
220	MetadataCollector_call, /* tp_call */
221	0, /* tp_str */
222	0, /* tp_getattro */
223	0, /* tp_setattro */
224	0, /* tp_as_buffer */
225	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
226	"MetadataCollectorUtility(runtime: Runtime \| None = None)\n"
227	"--\n\n"
228	"Collect metadata from a DFTracer trace file.\n\n"
229	"Args:\n"
230	" runtime (Runtime or None): Runtime for thread pool control.\n"
231	"\n"
232	"process(file_path, index_dir='') -> dict\n"
233	" file_path (str): Path to the trace file.\n"
234	" index_dir (str): Directory for index sidecar files.\n",
235	0, /* tp_traverse */
236	0, /* tp_clear */
237	0, /* tp_richcompare */
238	0, /* tp_weaklistoffset */
239	0, /* tp_iter */
240	0, /* tp_iternext */
241	MetadataCollector_methods, /* tp_methods */
242	0, /* tp_members */
243	0, /* tp_getset */
244	0, /* tp_base */
245	0, /* tp_dict */
246	0, /* tp_descr_get */
247	0, /* tp_descr_set */
248	0, /* tp_dictoffset */
249	(initproc)MetadataCollector_init, /* tp_init */
250	0, /* tp_alloc */
251	MetadataCollector_new, /* tp_new */
252	};
253
254	int init_metadata_collector(PyObject *m) {	1✔
255	if (PyType_Ready(&MetadataCollectorType) < 0) return -1;	1!
256
257	Py_INCREF(&MetadataCollectorType);	1✔
258	if (PyModule_AddObject(m, "MetadataCollectorUtility",	2!
259	(PyObject *)&MetadataCollectorType) < 0) {	1✔
UNCOV 260	Py_DECREF(&MetadataCollectorType);	×
UNCOV 261	Py_DECREF(m);	×
262	return -1;	×
263	}
264
265	return 0;	1✔
266	}	1✔

llnl / dftracer-utils / 23529483807

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous