26139121164

Committed 20 May 2026 03:18AM UTC coverage: 50.8% (-1.4%) from 52.2%

Build # 26139121164

Build Type

Pull #67

github

Committed by

web-flow

Commit Message

Merge 0979ebb80 into 6c9aaa7c9

Pull Request Pull Request #67: Added Clarion codes.

Coverage Stats

31484 of 79437 branches covered (39.63%)

Branch coverage included in aggregate %.

32397 of 46313 relevant lines covered (69.95%)

9978.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.74

/src/dftracer/utils/python/utilities/metadata_collector.cpp

#define PY_SSIZE_T_CLEAN
#include <dftracer/utils/core/common/archive_format.h>
#include <dftracer/utils/core/coro/task.h>
#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/metadata_collector.h>
#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
#include <dftracer/utils/utilities/composites/dft/metadata_collector_utility.h>

#include <string>

using dftracer::utils::get_format_name;
using dftracer::utils::Runtime;
using dftracer::utils::coro::CoroTask;
using namespace dftracer::utils::utilities::composites::dft;

static Runtime *get_runtime(MetadataCollectorObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void MetadataCollector_dealloc(MetadataCollectorObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *MetadataCollector_new(PyTypeObject *type, PyObject *args,
                                       PyObject *kwds) {
    MetadataCollectorObject *self =
        (MetadataCollectorObject *)type->tp_alloc(type, 0);
    if (self) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int MetadataCollector_init(MetadataCollectorObject *self, PyObject *args,
                                  PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *MetadataCollector_collect(MetadataCollectorObject *self,
                                           PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"file_path", "index_dir", NULL};
    const char *file_path;
    const char *index_dir = "";
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", (char **)kwlist,
                                     &file_path, &index_dir))
        return NULL;

    std::string file_path_str(file_path);
    std::string index_dir_str(index_dir);
    std::string error_msg;
    MetadataCollectorUtilityOutput output;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);

        MetadataCollectorUtilityInput input;
        input.file_path = file_path_str;
        input.index_path = dftracer::utils::utilities::composites::dft::
            internal::determine_index_path(file_path_str, index_dir_str);

        auto *out_p = &output;
        auto input_copy = input;
        auto task = [out_p, input_copy]() -> CoroTask<void> {
            MetadataCollectorUtility util;
            *out_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "metadata-collector").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    PyObject *d = PyDict_New();
    if (!d) return NULL;

#define SET_STR(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyUnicode_FromString(v);          \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_DBL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyFloat_FromDouble(v);            \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_SZT(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromSize_t(v);             \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_ULL(k, v)                                    \
    do {                                                 \
        PyObject *_v = PyLong_FromUnsignedLongLong(v);   \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

#define SET_BOOL(k, v)                                   \
    do {                                                 \
        PyObject *_v = PyBool_FromLong(v ? 1 : 0);       \
        if (!_v || PyDict_SetItemString(d, k, _v) < 0) { \
            Py_XDECREF(_v);                              \
            Py_DECREF(d);                                \
            return NULL;                                 \
        }                                                \
        Py_DECREF(_v);                                   \
    } while (0)

    SET_STR("file_path", output.file_path.c_str());
    SET_STR("index_path", output.index_path.c_str());
    SET_DBL("size_mb", output.size_mb);
    SET_SZT("start_line", output.start_line);
    SET_SZT("end_line", output.end_line);
    SET_SZT("valid_events", output.valid_events);
    SET_DBL("size_per_line", output.size_per_line);
    SET_BOOL("success", output.success);
    SET_BOOL("has_index", output.has_index);
    SET_BOOL("index_valid", output.index_valid);
    SET_ULL("compressed_size", output.compressed_size);
    SET_ULL("uncompressed_size", output.uncompressed_size);
    SET_ULL("num_lines", output.num_lines);
    SET_ULL("checkpoint_size", output.checkpoint_size);
    SET_SZT("num_checkpoints", output.num_checkpoints);
    SET_STR("format", get_format_name(output.format));
    SET_STR("error_message", output.error_message.c_str());

#undef SET_STR
#undef SET_DBL
#undef SET_SZT
#undef SET_ULL
#undef SET_BOOL

    return d;
}

static PyObject *MetadataCollector_call(PyObject *self, PyObject *args,
                                        PyObject *kwds) {
    return MetadataCollector_collect((MetadataCollectorObject *)self, args,
                                     kwds);
}

static PyMethodDef MetadataCollector_methods[] = {
    {"process", (PyCFunction)MetadataCollector_collect,
     METH_VARARGS | METH_KEYWORDS,
     "Collect metadata from a trace file.\n"
     "\n"
     "Args:\n"
     "    file_path (str): Path to the trace file.\n"
     "    index_dir (str): Directory for .dftindex stores.\n"},
    {NULL}};

PyTypeObject MetadataCollectorType = {
    PyVarObject_HEAD_INIT(
        NULL, 0) "dftracer_utils_ext.MetadataCollectorUtility", /* tp_name */
    sizeof(MetadataCollectorObject),          /* tp_basicsize */
    0,                                        /* tp_itemsize */
    (destructor)MetadataCollector_dealloc,    /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    MetadataCollector_call,                   /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "MetadataCollectorUtility(runtime: Runtime | None = None)\n"
    "--\n\n"
    "Collect metadata from a DFTracer trace file.\n\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "\n"
    "process(file_path, index_dir='') -> dict\n"
    "    file_path (str): Path to the trace file.\n"
    "    index_dir (str): Directory for .dftindex stores.\n",
    0,                                /* tp_traverse */
    0,                                /* tp_clear */
    0,                                /* tp_richcompare */
    0,                                /* tp_weaklistoffset */
    0,                                /* tp_iter */
    0,                                /* tp_iternext */
    MetadataCollector_methods,        /* tp_methods */
    0,                                /* tp_members */
    0,                                /* tp_getset */
    0,                                /* tp_base */
    0,                                /* tp_dict */
    0,                                /* tp_descr_get */
    0,                                /* tp_descr_set */
    0,                                /* tp_dictoffset */
    (initproc)MetadataCollector_init, /* tp_init */
    0,                                /* tp_alloc */
    MetadataCollector_new,            /* tp_new */
};

int init_metadata_collector(PyObject *m) {
    if (PyType_Ready(&MetadataCollectorType) < 0) return -1;

    Py_INCREF(&MetadataCollectorType);
    if (PyModule_AddObject(m, "MetadataCollectorUtility",
                           (PyObject *)&MetadataCollectorType) < 0) {
        Py_DECREF(&MetadataCollectorType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/core/common/archive_format.h>
3	#include <dftracer/utils/core/coro/task.h>
4	#include <dftracer/utils/core/runtime.h>
5	#include <dftracer/utils/python/runtime.h>
6	#include <dftracer/utils/python/utilities/metadata_collector.h>
7	#include <dftracer/utils/utilities/composites/dft/internal/utils.h>
8	#include <dftracer/utils/utilities/composites/dft/metadata_collector_utility.h>
9
10	#include <string>
11
12	using dftracer::utils::get_format_name;
13	using dftracer::utils::Runtime;
14	using dftracer::utils::coro::CoroTask;
15	using namespace dftracer::utils::utilities::composites::dft;
16
17	static Runtime get_runtime(MetadataCollectorObject self) {	4✔
18	if (self->runtime_obj)	4!
19	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
20	return get_default_runtime();	4✔
21	}	4✔
22
23	static void MetadataCollector_dealloc(MetadataCollectorObject *self) {	4✔
24	Py_XDECREF(self->runtime_obj);	4✔
25	Py_TYPE(self)->tp_free((PyObject *)self);	4✔
26	}	4✔
27
28	static PyObject MetadataCollector_new(PyTypeObject type, PyObject *args,	4✔
29	PyObject *kwds) {
30	MetadataCollectorObject *self =	4✔
31	(MetadataCollectorObject *)type->tp_alloc(type, 0);	4✔
32	if (self) {	4!
33	self->runtime_obj = NULL;	4✔
34	}	4✔
35	return (PyObject *)self;	4✔
36	}
37
38	static int MetadataCollector_init(MetadataCollectorObject self, PyObject args,	4✔
39	PyObject *kwds) {
40	static const char *kwlist[] = {"runtime", NULL};
41	PyObject *runtime_arg = NULL;	4✔
42
43	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	4!
44	&runtime_arg)) {
45	return -1;	×
46	}
47
48	if (runtime_arg && runtime_arg != Py_None) {	4!
49	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
50	Py_INCREF(runtime_arg);	×
51	self->runtime_obj = runtime_arg;	×
52	} else {	×
53	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
54	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
55	self->runtime_obj = native;	×
56	} else {	×
57	Py_XDECREF(native);	×
58	PyErr_SetString(PyExc_TypeError,	×
59	"runtime must be a Runtime instance or None");
60	return -1;	×
61	}
62	}
63	}	×
64
65	return 0;	4✔
66	}	4✔
67
68	static PyObject MetadataCollector_collect(MetadataCollectorObject self,	4✔
69	PyObject args, PyObject kwds) {
70	static const char *kwlist[] = {"file_path", "index_dir", NULL};
71	const char *file_path;
72	const char *index_dir = "";	4✔
73	if (!PyArg_ParseTupleAndKeywords(args, kwds, "s\|s", (char **)kwlist,	4!
74	&file_path, &index_dir))
75	return NULL;	×
76
77	std::string file_path_str(file_path);	4✔
78	std::string index_dir_str(index_dir);	4!
79	std::string error_msg;	4✔
80	MetadataCollectorUtilityOutput output;	4✔
81
82	Py_BEGIN_ALLOW_THREADS try {	4!
83	Runtime *rt = get_runtime(self);	4!
84
85	MetadataCollectorUtilityInput input;	4✔
86	input.file_path = file_path_str;	4!
87	input.index_path = dftracer::utils::utilities::composites::dft::	4!
88	internal::determine_index_path(file_path_str, index_dir_str);
89
90	auto *out_p = &output;	4✔
91	auto input_copy = input;	4!
92	auto task = [out_p, input_copy]() -> CoroTask<void> {	32!
93	MetadataCollectorUtility util;	12!
94	*out_p = co_await util.process(input_copy);	16!
95	};	12!
96	rt->submit(task(), "metadata-collector").get();	4!
97	} catch (const std::exception &e) {	4!
98	error_msg = e.what();	×
99	}	×
100	Py_END_ALLOW_THREADS	4!
101
102	if (!error_msg.empty()) {	4!
103	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
104	return NULL;	×
105	}
106
107	PyObject *d = PyDict_New();	4!
108	if (!d) return NULL;	4!
109
110	#define SET_STR(k, v) \
111	do { \
112	PyObject *_v = PyUnicode_FromString(v); \
113	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
114	Py_XDECREF(_v); \
115	Py_DECREF(d); \
116	return NULL; \
117	} \
118	Py_DECREF(_v); \
119	} while (0)
120
121	#define SET_DBL(k, v) \
122	do { \
123	PyObject *_v = PyFloat_FromDouble(v); \
124	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
125	Py_XDECREF(_v); \
126	Py_DECREF(d); \
127	return NULL; \
128	} \
129	Py_DECREF(_v); \
130	} while (0)
131
132	#define SET_SZT(k, v) \
133	do { \
134	PyObject *_v = PyLong_FromSize_t(v); \
135	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
136	Py_XDECREF(_v); \
137	Py_DECREF(d); \
138	return NULL; \
139	} \
140	Py_DECREF(_v); \
141	} while (0)
142
143	#define SET_ULL(k, v) \
144	do { \
145	PyObject *_v = PyLong_FromUnsignedLongLong(v); \
146	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
147	Py_XDECREF(_v); \
148	Py_DECREF(d); \
149	return NULL; \
150	} \
151	Py_DECREF(_v); \
152	} while (0)
153
154	#define SET_BOOL(k, v) \
155	do { \
156	PyObject *_v = PyBool_FromLong(v ? 1 : 0); \
157	if (!_v \|\| PyDict_SetItemString(d, k, _v) < 0) { \
158	Py_XDECREF(_v); \
159	Py_DECREF(d); \
160	return NULL; \
161	} \
162	Py_DECREF(_v); \
163	} while (0)
164
165	SET_STR("file_path", output.file_path.c_str());	4!
166	SET_STR("index_path", output.index_path.c_str());	4!
167	SET_DBL("size_mb", output.size_mb);	4!
168	SET_SZT("start_line", output.start_line);	4!
169	SET_SZT("end_line", output.end_line);	4!
170	SET_SZT("valid_events", output.valid_events);	4!
171	SET_DBL("size_per_line", output.size_per_line);	4!
172	SET_BOOL("success", output.success);	4!
173	SET_BOOL("has_index", output.has_index);	4!
174	SET_BOOL("index_valid", output.index_valid);	4!
175	SET_ULL("compressed_size", output.compressed_size);	4!
176	SET_ULL("uncompressed_size", output.uncompressed_size);	4!
177	SET_ULL("num_lines", output.num_lines);	4!
178	SET_ULL("checkpoint_size", output.checkpoint_size);	4!
179	SET_SZT("num_checkpoints", output.num_checkpoints);	4!
180	SET_STR("format", get_format_name(output.format));	4!
181	SET_STR("error_message", output.error_message.c_str());	4!
182
183	#undef SET_STR
184	#undef SET_DBL
185	#undef SET_SZT
186	#undef SET_ULL
187	#undef SET_BOOL
188
189	return d;	4✔
190	}	4✔
191
192	static PyObject MetadataCollector_call(PyObject self, PyObject *args,	1✔
193	PyObject *kwds) {
194	return MetadataCollector_collect((MetadataCollectorObject *)self, args,	2✔
195	kwds);	1✔
196	}
197
198	static PyMethodDef MetadataCollector_methods[] = {
199	{"process", (PyCFunction)MetadataCollector_collect,
200	METH_VARARGS \| METH_KEYWORDS,
201	"Collect metadata from a trace file.\n"
202	"\n"
203	"Args:\n"
204	" file_path (str): Path to the trace file.\n"
205	" index_dir (str): Directory for .dftindex stores.\n"},
206	{NULL}};
207
208	PyTypeObject MetadataCollectorType = {
209	PyVarObject_HEAD_INIT(
210	NULL, 0) "dftracer_utils_ext.MetadataCollectorUtility", /* tp_name */
211	sizeof(MetadataCollectorObject), /* tp_basicsize */
212	0, /* tp_itemsize */
213	(destructor)MetadataCollector_dealloc, /* tp_dealloc */
214	0, /* tp_vectorcall_offset */
215	0, /* tp_getattr */
216	0, /* tp_setattr */
217	0, /* tp_as_async */
218	0, /* tp_repr */
219	0, /* tp_as_number */
220	0, /* tp_as_sequence */
221	0, /* tp_as_mapping */
222	0, /* tp_hash */
223	MetadataCollector_call, /* tp_call */
224	0, /* tp_str */
225	0, /* tp_getattro */
226	0, /* tp_setattro */
227	0, /* tp_as_buffer */
228	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
229	"MetadataCollectorUtility(runtime: Runtime \| None = None)\n"
230	"--\n\n"
231	"Collect metadata from a DFTracer trace file.\n\n"
232	"Args:\n"
233	" runtime (Runtime or None): Runtime for thread pool control.\n"
234	"\n"
235	"process(file_path, index_dir='') -> dict\n"
236	" file_path (str): Path to the trace file.\n"
237	" index_dir (str): Directory for .dftindex stores.\n",
238	0, /* tp_traverse */
239	0, /* tp_clear */
240	0, /* tp_richcompare */
241	0, /* tp_weaklistoffset */
242	0, /* tp_iter */
243	0, /* tp_iternext */
244	MetadataCollector_methods, /* tp_methods */
245	0, /* tp_members */
246	0, /* tp_getset */
247	0, /* tp_base */
248	0, /* tp_dict */
249	0, /* tp_descr_get */
250	0, /* tp_descr_set */
251	0, /* tp_dictoffset */
252	(initproc)MetadataCollector_init, /* tp_init */
253	0, /* tp_alloc */
254	MetadataCollector_new, /* tp_new */
255	};
256
257	int init_metadata_collector(PyObject *m) {	1✔
258	if (PyType_Ready(&MetadataCollectorType) < 0) return -1;	1!
259
260	Py_INCREF(&MetadataCollectorType);	1✔
261	if (PyModule_AddObject(m, "MetadataCollectorUtility",	2!
262	(PyObject *)&MetadataCollectorType) < 0) {	1✔
263	Py_DECREF(&MetadataCollectorType);	×
264	Py_DECREF(m);	×
265	return -1;	×
266	}
267
268	return 0;	1✔
269	}	1✔

llnl / dftracer-utils / 26139121164

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous