27052412546

Committed 06 Jun 2026 04:20AM UTC coverage: 50.862% (+1.0%) from 49.905%

Build # 27052412546

Build Type

Pull #73

github

Committed by

web-flow

Commit Message

Merge 734572730 into 88a3c8457

Pull Request Pull Request #73: add portable dependencies wheel support

Coverage Stats

31801 of 79859 branches covered (39.82%)

Branch coverage included in aggregate %.

32491 of 46545 relevant lines covered (69.81%)

9947.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.38

/src/dftracer/utils/python/utilities/reconstruction_planner.cpp

#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/reconstruction_planner.h>
#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>

#include <string>
#include <vector>

using dftracer::utils::Runtime;
using namespace dftracer::utils::utilities::composites::dft::reorganize;

static Runtime *get_runtime(ReconstructionPlannerObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *ReconstructionPlanner_new(PyTypeObject *type, PyObject *args,
                                           PyObject *kwds) {
    ReconstructionPlannerObject *self;
    self = (ReconstructionPlannerObject *)type->tp_alloc(type, 0);
    if (self != NULL) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,
                                      PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *ReconstructionPlanner_plan(ReconstructionPlannerObject *self,
                                            PyObject *args, PyObject *kwds) {
    using dftracer::utils::coro::CoroTask;

    static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
    PyObject *files_obj;
    const char *index_dir = "";

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s", (char **)kwlist,
                                     &files_obj, &index_dir))
        return NULL;

    if (!PyList_Check(files_obj)) {
        PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");
        return NULL;
    }

    Py_ssize_t nfiles = PyList_Size(files_obj);
    std::vector<std::string> files;
    files.reserve(static_cast<std::size_t>(nfiles));
    for (Py_ssize_t i = 0; i < nfiles; i++) {
        PyObject *item = PyList_GetItem(files_obj, i);
        const char *s = PyUnicode_AsUTF8(item);
        if (!s) return NULL;
        files.emplace_back(s);
    }

    ReconstructionPlannerInput input;
    input.reorganized_files = std::move(files);
    input.index_dir = index_dir;

    ReconstructionPlan plan;
    auto *plan_p = &plan;
    ReconstructionPlannerInput input_copy = input;
    std::string error_msg;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);
        auto task = [plan_p, input_copy]() -> CoroTask<void> {
            ReconstructionPlannerUtility util;
            *plan_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "reconstruction-planner").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    // files dict: original_path -> reconstruction info
    PyObject *py_files = PyDict_New();
    if (!py_files) return NULL;

    for (const auto &[orig_path, recon] : plan.files) {
        // checkpoint_segments: int -> list of segment dicts
        PyObject *py_segs = PyDict_New();
        if (!py_segs) {
            Py_DECREF(py_files);
            return NULL;
        }

        for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {
            PyObject *py_seg_list =
                PyList_New(static_cast<Py_ssize_t>(seg_list.size()));
            if (!py_seg_list) {
                Py_DECREF(py_segs);
                Py_DECREF(py_files);
                return NULL;
            }
            for (std::size_t si = 0; si < seg_list.size(); si++) {
                const auto &seg = seg_list[si];
                PyObject *sd = PyDict_New();
                if (!sd) {
                    Py_DECREF(py_seg_list);
                    Py_DECREF(py_segs);
                    Py_DECREF(py_files);
                    return NULL;
                }
                PyDict_SetItemString(
                    sd, "reorg_file",
                    PyUnicode_FromString(seg.reorg_file.c_str()));
                PyDict_SetItemString(sd, "output_line_start",
                                     PyLong_FromLong(seg.output_line_start));
                PyDict_SetItemString(sd, "output_line_end",
                                     PyLong_FromLong(seg.output_line_end));
                PyDict_SetItemString(sd, "source_checkpoint",
                                     PyLong_FromLong(seg.source_checkpoint));
                PyDict_SetItemString(sd, "event_count",
                                     PyLong_FromLong(seg.event_count));
                PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);
            }
            PyObject *py_cp_key = PyLong_FromLong(cp_idx);
            PyDict_SetItem(py_segs, py_cp_key, py_seg_list);
            Py_DECREF(py_cp_key);
            Py_DECREF(py_seg_list);
        }

        PyObject *py_recon = PyDict_New();
        if (!py_recon) {
            Py_DECREF(py_segs);
            Py_DECREF(py_files);
            return NULL;
        }
        PyDict_SetItemString(py_recon, "original_path",
                             PyUnicode_FromString(recon.original_path.c_str()));
        PyDict_SetItemString(py_recon, "num_checkpoints",
                             PyLong_FromLong(recon.num_checkpoints));
        PyDict_SetItemString(py_recon, "event_hash",
                             PyUnicode_FromString(recon.event_hash.c_str()));
        PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);
        Py_DECREF(py_segs);

        PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);
        Py_DECREF(py_recon);
    }

    PyObject *result = PyDict_New();
    if (!result) {
        Py_DECREF(py_files);
        return NULL;
    }
    PyDict_SetItemString(result, "files", py_files);
    Py_DECREF(py_files);
    PyDict_SetItemString(result, "total_segments",
                         PyLong_FromSize_t(plan.total_segments));
    PyDict_SetItemString(result, "total_events",
                         PyLong_FromSize_t(plan.total_events));
    return result;
}

static PyObject *ReconstructionPlanner_call(PyObject *self, PyObject *args,
                                            PyObject *kwds) {
    return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,
                                      kwds);
}

static PyMethodDef ReconstructionPlanner_methods[] = {
    {"process", (PyCFunction)ReconstructionPlanner_plan,
     METH_VARARGS | METH_KEYWORDS,
     "process(reorganized_files, index_dir='')\n"
     "--\n"
     "\n"
     "Build a reconstruction plan from reorganized files.\n"
     "\n"
     "Args:\n"
     "    reorganized_files (list[str]): Paths to reorganized files.\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Reconstruction plan.\n"},
    {NULL} /* Sentinel */
};

PyTypeObject ReconstructionPlannerType = {
    PyVarObject_HEAD_INIT(
        NULL,
        0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
    sizeof(ReconstructionPlannerObject),                      /* tp_basicsize */
    0,                                                        /* tp_itemsize */
    (destructor)ReconstructionPlanner_dealloc,                /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    ReconstructionPlanner_call,               /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "ReconstructionPlannerUtility(runtime: Runtime | None = None)\n"
    "--\n"
    "\n"
    "Plan reconstruction of original files from reorganized trace files.\n"
    "\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "        If None, uses the default global Runtime.\n"
    "\n"
    "process(reorganized_files, index_dir='') -> dict\n"
    "    reorganized_files (list[str]): Paths to reorganized trace files.\n"
    "    index_dir (str): Directory containing `.dftindex` stores.\n",
    /* tp_doc */
    0,                                    /* tp_traverse */
    0,                                    /* tp_clear */
    0,                                    /* tp_richcompare */
    0,                                    /* tp_weaklistoffset */
    0,                                    /* tp_iter */
    0,                                    /* tp_iternext */
    ReconstructionPlanner_methods,        /* tp_methods */
    0,                                    /* tp_members */
    0,                                    /* tp_getset */
    0,                                    /* tp_base */
    0,                                    /* tp_dict */
    0,                                    /* tp_descr_get */
    0,                                    /* tp_descr_set */
    0,                                    /* tp_dictoffset */
    (initproc)ReconstructionPlanner_init, /* tp_init */
    0,                                    /* tp_alloc */
    ReconstructionPlanner_new,            /* tp_new */
};

int init_reconstruction_planner(PyObject *m) {
    if (PyType_Ready(&ReconstructionPlannerType) < 0) return -1;

    Py_INCREF(&ReconstructionPlannerType);
    if (PyModule_AddObject(m, "ReconstructionPlannerUtility",
                           (PyObject *)&ReconstructionPlannerType) < 0) {
        Py_DECREF(&ReconstructionPlannerType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#include <dftracer/utils/core/runtime.h>
2	#include <dftracer/utils/python/runtime.h>
3	#include <dftracer/utils/python/utilities/reconstruction_planner.h>
4	#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>
5
6	#include <string>
7	#include <vector>
8
9	using dftracer::utils::Runtime;
10	using namespace dftracer::utils::utilities::composites::dft::reorganize;
11
12	static Runtime get_runtime(ReconstructionPlannerObject self) {	2✔
13	if (self->runtime_obj)	2!
14	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
15	return get_default_runtime();	2✔
16	}	2✔
17
18	static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {	2✔
19	Py_XDECREF(self->runtime_obj);	2✔
20	Py_TYPE(self)->tp_free((PyObject *)self);	2✔
21	}	2✔
22
23	static PyObject ReconstructionPlanner_new(PyTypeObject type, PyObject *args,	2✔
24	PyObject *kwds) {
25	ReconstructionPlannerObject *self;
26	self = (ReconstructionPlannerObject *)type->tp_alloc(type, 0);	2✔
27	if (self != NULL) {	2!
28	self->runtime_obj = NULL;	2✔
29	}	2✔
30	return (PyObject *)self;	2✔
31	}
32
33	static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,	2✔
34	PyObject args, PyObject kwds) {
35	static const char *kwlist[] = {"runtime", NULL};
36	PyObject *runtime_arg = NULL;	2✔
37
38	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	2!
39	&runtime_arg)) {
40	return -1;	×
41	}
42
43	if (runtime_arg && runtime_arg != Py_None) {	2!
44	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
45	Py_INCREF(runtime_arg);	×
46	self->runtime_obj = runtime_arg;	×
47	} else {	×
48	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
49	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
50	self->runtime_obj = native;	×
51	} else {	×
52	Py_XDECREF(native);	×
53	PyErr_SetString(PyExc_TypeError,	×
54	"runtime must be a Runtime instance or None");
55	return -1;	×
56	}
57	}
58	}	×
59
60	return 0;	2✔
61	}	2✔
62
63	static PyObject ReconstructionPlanner_plan(ReconstructionPlannerObject self,	2✔
64	PyObject args, PyObject kwds) {
65	using dftracer::utils::coro::CoroTask;
66
67	static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
68	PyObject *files_obj;
69	const char *index_dir = "";	2✔
70
71	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O\|s", (char **)kwlist,	2!
72	&files_obj, &index_dir))
73	return NULL;	×
74
75	if (!PyList_Check(files_obj)) {	2!
76	PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");	×
77	return NULL;	×
78	}
79
80	Py_ssize_t nfiles = PyList_Size(files_obj);	2✔
81	std::vector<std::string> files;	2✔
82	files.reserve(static_cast<std::size_t>(nfiles));	2!
83	for (Py_ssize_t i = 0; i < nfiles; i++) {	2!
84	PyObject *item = PyList_GetItem(files_obj, i);	×
85	const char *s = PyUnicode_AsUTF8(item);	×
86	if (!s) return NULL;	×
87	files.emplace_back(s);	×
88	}	×
89
90	ReconstructionPlannerInput input;	2✔
91	input.reorganized_files = std::move(files);	2✔
92	input.index_dir = index_dir;	2!
93
94	ReconstructionPlan plan;	2✔
95	auto *plan_p = &plan;	2✔
96	ReconstructionPlannerInput input_copy = input;	2!
97	std::string error_msg;	2✔
98
99	Py_BEGIN_ALLOW_THREADS try {	2!
100	Runtime *rt = get_runtime(self);	2!
101	auto task = [plan_p, input_copy]() -> CoroTask<void> {	16!
102	ReconstructionPlannerUtility util;	6!
103	*plan_p = co_await util.process(input_copy);	8!
104	};	6!
105	rt->submit(task(), "reconstruction-planner").get();	2!
106	} catch (const std::exception &e) {	2!
107	error_msg = e.what();	×
108	}	×
109	Py_END_ALLOW_THREADS	2!
110
111	if (!error_msg.empty()) {	2!
112	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
113	return NULL;	×
114	}
115
116	// files dict: original_path -> reconstruction info
117	PyObject *py_files = PyDict_New();	2!
118	if (!py_files) return NULL;	2!
119
120	for (const auto &[orig_path, recon] : plan.files) {	2!
121	// checkpoint_segments: int -> list of segment dicts
122	PyObject *py_segs = PyDict_New();	×
123	if (!py_segs) {	×
124	Py_DECREF(py_files);	×
125	return NULL;	×
126	}
127
128	for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {	×
129	PyObject *py_seg_list =	×
130	PyList_New(static_cast<Py_ssize_t>(seg_list.size()));	×
131	if (!py_seg_list) {	×
132	Py_DECREF(py_segs);	×
133	Py_DECREF(py_files);	×
134	return NULL;	×
135	}
136	for (std::size_t si = 0; si < seg_list.size(); si++) {	×
137	const auto &seg = seg_list[si];	×
138	PyObject *sd = PyDict_New();	×
139	if (!sd) {	×
140	Py_DECREF(py_seg_list);	×
141	Py_DECREF(py_segs);	×
142	Py_DECREF(py_files);	×
143	return NULL;	×
144	}
145	PyDict_SetItemString(	×
146	sd, "reorg_file",	×
147	PyUnicode_FromString(seg.reorg_file.c_str()));	×
148	PyDict_SetItemString(sd, "output_line_start",	×
149	PyLong_FromLong(seg.output_line_start));	×
150	PyDict_SetItemString(sd, "output_line_end",	×
151	PyLong_FromLong(seg.output_line_end));	×
152	PyDict_SetItemString(sd, "source_checkpoint",	×
153	PyLong_FromLong(seg.source_checkpoint));	×
154	PyDict_SetItemString(sd, "event_count",	×
155	PyLong_FromLong(seg.event_count));	×
156	PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);	×
157	}	×
158	PyObject *py_cp_key = PyLong_FromLong(cp_idx);	×
159	PyDict_SetItem(py_segs, py_cp_key, py_seg_list);	×
160	Py_DECREF(py_cp_key);	×
161	Py_DECREF(py_seg_list);	×
162	}
163
164	PyObject *py_recon = PyDict_New();	×
165	if (!py_recon) {	×
166	Py_DECREF(py_segs);	×
167	Py_DECREF(py_files);	×
168	return NULL;	×
169	}
170	PyDict_SetItemString(py_recon, "original_path",	×
171	PyUnicode_FromString(recon.original_path.c_str()));	×
172	PyDict_SetItemString(py_recon, "num_checkpoints",	×
173	PyLong_FromLong(recon.num_checkpoints));	×
174	PyDict_SetItemString(py_recon, "event_hash",	×
175	PyUnicode_FromString(recon.event_hash.c_str()));	×
176	PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);	×
177	Py_DECREF(py_segs);	×
178
179	PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);	×
180	Py_DECREF(py_recon);	×
181	}
182
183	PyObject *result = PyDict_New();	2!
184	if (!result) {	2!
185	Py_DECREF(py_files);	×
186	return NULL;	×
187	}
188	PyDict_SetItemString(result, "files", py_files);	2!
189	Py_DECREF(py_files);	2!
190	PyDict_SetItemString(result, "total_segments",	2!
191	PyLong_FromSize_t(plan.total_segments));	2!
192	PyDict_SetItemString(result, "total_events",	2!
193	PyLong_FromSize_t(plan.total_events));	2!
194	return result;	2✔
195	}	2✔
196
197	static PyObject ReconstructionPlanner_call(PyObject self, PyObject *args,	1✔
198	PyObject *kwds) {
199	return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,	2✔
200	kwds);	1✔
201	}
202
203	static PyMethodDef ReconstructionPlanner_methods[] = {
204	{"process", (PyCFunction)ReconstructionPlanner_plan,
205	METH_VARARGS \| METH_KEYWORDS,
206	"process(reorganized_files, index_dir='')\n"
207	"--\n"
208	"\n"
209	"Build a reconstruction plan from reorganized files.\n"
210	"\n"
211	"Args:\n"
212	" reorganized_files (list[str]): Paths to reorganized files.\n"
213	" index_dir (str): Directory for .dftindex stores (default '').\n"
214	"\n"
215	"Returns:\n"
216	" dict: Reconstruction plan.\n"},
217	{NULL} /* Sentinel */
218	};
219
220	PyTypeObject ReconstructionPlannerType = {
221	PyVarObject_HEAD_INIT(
222	NULL,
223	0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
224	sizeof(ReconstructionPlannerObject), /* tp_basicsize */
225	0, /* tp_itemsize */
226	(destructor)ReconstructionPlanner_dealloc, /* tp_dealloc */
227	0, /* tp_vectorcall_offset */
228	0, /* tp_getattr */
229	0, /* tp_setattr */
230	0, /* tp_as_async */
231	0, /* tp_repr */
232	0, /* tp_as_number */
233	0, /* tp_as_sequence */
234	0, /* tp_as_mapping */
235	0, /* tp_hash */
236	ReconstructionPlanner_call, /* tp_call */
237	0, /* tp_str */
238	0, /* tp_getattro */
239	0, /* tp_setattro */
240	0, /* tp_as_buffer */
241	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
242	"ReconstructionPlannerUtility(runtime: Runtime \| None = None)\n"
243	"--\n"
244	"\n"
245	"Plan reconstruction of original files from reorganized trace files.\n"
246	"\n"
247	"Args:\n"
248	" runtime (Runtime or None): Runtime for thread pool control.\n"
249	" If None, uses the default global Runtime.\n"
250	"\n"
251	"process(reorganized_files, index_dir='') -> dict\n"
252	" reorganized_files (list[str]): Paths to reorganized trace files.\n"
253	" index_dir (str): Directory containing `.dftindex` stores.\n",
254	/* tp_doc */
255	0, /* tp_traverse */
256	0, /* tp_clear */
257	0, /* tp_richcompare */
258	0, /* tp_weaklistoffset */
259	0, /* tp_iter */
260	0, /* tp_iternext */
261	ReconstructionPlanner_methods, /* tp_methods */
262	0, /* tp_members */
263	0, /* tp_getset */
264	0, /* tp_base */
265	0, /* tp_dict */
266	0, /* tp_descr_get */
267	0, /* tp_descr_set */
268	0, /* tp_dictoffset */
269	(initproc)ReconstructionPlanner_init, /* tp_init */
270	0, /* tp_alloc */
271	ReconstructionPlanner_new, /* tp_new */
272	};
273
274	int init_reconstruction_planner(PyObject *m) {	1✔
275	if (PyType_Ready(&ReconstructionPlannerType) < 0) return -1;	1!
276
277	Py_INCREF(&ReconstructionPlannerType);	1✔
278	if (PyModule_AddObject(m, "ReconstructionPlannerUtility",	2!
279	(PyObject *)&ReconstructionPlannerType) < 0) {	1✔
280	Py_DECREF(&ReconstructionPlannerType);	×
281	Py_DECREF(m);	×
282	return -1;	×
283	}
284
285	return 0;	1✔
286	}	1✔

llnl / dftracer-utils / 27052412546

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous