27739884676

Committed 18 Jun 2026 05:57AM UTC coverage: 52.155% (+0.04%) from 52.111%

Build # 27739884676

Build Type

Pull #79

github

Committed by

web-flow

Commit Message

Merge 93a88987f into 53ad1e86c

Pull Request Pull Request #79: Add Valgrind memory checking (C++, Python, MPI) and fix the bugs it found

Coverage Stats

37169 of 92677 branches covered (40.11%)

Branch coverage included in aggregate %.

129 of 144 new or added lines in 11 files covered. (89.58%)

7 existing lines in 3 files now uncovered.

33450 of 42726 relevant lines covered (78.29%)

20413.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.25

/src/dftracer/utils/python/utilities/reconstruction_planner.cpp

#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/py_dict_helpers.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/reconstruction_planner.h>
#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>

#include <string>
#include <vector>

using dftracer::utils::Runtime;
using namespace dftracer::utils::utilities::composites::dft::reorganize;

static Runtime *get_runtime(ReconstructionPlannerObject *self) {
    if (self->runtime_obj)
        return ((RuntimeObject *)self->runtime_obj)->runtime.get();
    return get_default_runtime();
}

static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {
    Py_XDECREF(self->runtime_obj);
    Py_TYPE(self)->tp_free((PyObject *)self);
}

static PyObject *ReconstructionPlanner_new(PyTypeObject *type, PyObject *args,
                                           PyObject *kwds) {
    ReconstructionPlannerObject *self;
    self = (ReconstructionPlannerObject *)type->tp_alloc(type, 0);
    if (self != NULL) {
        self->runtime_obj = NULL;
    }
    return (PyObject *)self;
}

static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,
                                      PyObject *args, PyObject *kwds) {
    static const char *kwlist[] = {"runtime", NULL};
    PyObject *runtime_arg = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", (char **)kwlist,
                                     &runtime_arg)) {
        return -1;
    }

    if (runtime_arg && runtime_arg != Py_None) {
        if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {
            Py_INCREF(runtime_arg);
            self->runtime_obj = runtime_arg;
        } else {
            PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");
            if (native && PyObject_TypeCheck(native, &RuntimeType)) {
                self->runtime_obj = native;
            } else {
                Py_XDECREF(native);
                PyErr_SetString(PyExc_TypeError,
                                "runtime must be a Runtime instance or None");
                return -1;
            }
        }
    }

    return 0;
}

static PyObject *ReconstructionPlanner_plan(ReconstructionPlannerObject *self,
                                            PyObject *args, PyObject *kwds) {
    using dftracer::utils::coro::CoroTask;

    static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
    PyObject *files_obj;
    const char *index_dir = "";

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s", (char **)kwlist,
                                     &files_obj, &index_dir))
        return NULL;

    if (!PyList_Check(files_obj)) {
        PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");
        return NULL;
    }

    Py_ssize_t nfiles = PyList_Size(files_obj);
    std::vector<std::string> files;
    files.reserve(static_cast<std::size_t>(nfiles));
    for (Py_ssize_t i = 0; i < nfiles; i++) {
        PyObject *item = PyList_GetItem(files_obj, i);
        const char *s = PyUnicode_AsUTF8(item);
        if (!s) return NULL;
        files.emplace_back(s);
    }

    ReconstructionPlannerInput input;
    input.reorganized_files = std::move(files);
    input.index_dir = index_dir;

    ReconstructionPlan plan;
    auto *plan_p = &plan;
    ReconstructionPlannerInput input_copy = input;
    std::string error_msg;

    Py_BEGIN_ALLOW_THREADS try {
        Runtime *rt = get_runtime(self);
        auto task = [plan_p, input_copy]() -> CoroTask<void> {
            ReconstructionPlannerUtility util;
            *plan_p = co_await util.process(input_copy);
        };
        rt->submit(task(), "reconstruction-planner").get();
    } catch (const std::exception &e) {
        error_msg = e.what();
    }
    Py_END_ALLOW_THREADS

        if (!error_msg.empty()) {
        PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());
        return NULL;
    }

    // files dict: original_path -> reconstruction info
    PyObject *py_files = PyDict_New();
    if (!py_files) return NULL;

    for (const auto &[orig_path, recon] : plan.files) {
        // checkpoint_segments: int -> list of segment dicts
        PyObject *py_segs = PyDict_New();
        if (!py_segs) {
            Py_DECREF(py_files);
            return NULL;
        }

        for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {
            PyObject *py_seg_list =
                PyList_New(static_cast<Py_ssize_t>(seg_list.size()));
            if (!py_seg_list) {
                Py_DECREF(py_segs);
                Py_DECREF(py_files);
                return NULL;
            }
            for (std::size_t si = 0; si < seg_list.size(); si++) {
                const auto &seg = seg_list[si];
                PyObject *sd = PyDict_New();
                if (!sd) {
                    Py_DECREF(py_seg_list);
                    Py_DECREF(py_segs);
                    Py_DECREF(py_files);
                    return NULL;
                }
                dict_set_steal(sd, "reorg_file",
                               PyUnicode_FromString(seg.reorg_file.c_str()));
                dict_set_steal(sd, "output_line_start",
                               PyLong_FromLong(seg.output_line_start));
                dict_set_steal(sd, "output_line_end",
                               PyLong_FromLong(seg.output_line_end));
                dict_set_steal(sd, "source_checkpoint",
                               PyLong_FromLong(seg.source_checkpoint));
                dict_set_steal(sd, "event_count",
                               PyLong_FromLong(seg.event_count));
                PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);
            }
            PyObject *py_cp_key = PyLong_FromLong(cp_idx);
            PyDict_SetItem(py_segs, py_cp_key, py_seg_list);
            Py_DECREF(py_cp_key);
            Py_DECREF(py_seg_list);
        }

        PyObject *py_recon = PyDict_New();
        if (!py_recon) {
            Py_DECREF(py_segs);
            Py_DECREF(py_files);
            return NULL;
        }
        dict_set_steal(py_recon, "original_path",
                       PyUnicode_FromString(recon.original_path.c_str()));
        dict_set_steal(py_recon, "num_checkpoints",
                       PyLong_FromLong(recon.num_checkpoints));
        dict_set_steal(py_recon, "event_hash",
                       PyUnicode_FromString(recon.event_hash.c_str()));
        PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);
        Py_DECREF(py_segs);

        PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);
        Py_DECREF(py_recon);
    }

    PyObject *result = PyDict_New();
    if (!result) {
        Py_DECREF(py_files);
        return NULL;
    }
    PyDict_SetItemString(result, "files", py_files);
    Py_DECREF(py_files);
    dict_set_steal(result, "total_segments",
                   PyLong_FromSize_t(plan.total_segments));
    dict_set_steal(result, "total_events",
                   PyLong_FromSize_t(plan.total_events));
    return result;
}

static PyObject *ReconstructionPlanner_call(PyObject *self, PyObject *args,
                                            PyObject *kwds) {
    return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,
                                      kwds);
}

static PyMethodDef ReconstructionPlanner_methods[] = {
    {"process", (PyCFunction)ReconstructionPlanner_plan,
     METH_VARARGS | METH_KEYWORDS,
     "process(reorganized_files, index_dir='')\n"
     "--\n"
     "\n"
     "Build a reconstruction plan from reorganized files.\n"
     "\n"
     "Args:\n"
     "    reorganized_files (list[str]): Paths to reorganized files.\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Reconstruction plan.\n"},
    {NULL} /* Sentinel */
};

PyTypeObject ReconstructionPlannerType = {
    PyVarObject_HEAD_INIT(
        NULL,
        0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
    sizeof(ReconstructionPlannerObject),                      /* tp_basicsize */
    0,                                                        /* tp_itemsize */
    (destructor)ReconstructionPlanner_dealloc,                /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    ReconstructionPlanner_call,               /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "ReconstructionPlannerUtility(runtime: Runtime | None = None)\n"
    "--\n"
    "\n"
    "Plan reconstruction of original files from reorganized trace files.\n"
    "\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "        If None, uses the default global Runtime.\n"
    "\n"
    "process(reorganized_files, index_dir='') -> dict\n"
    "    reorganized_files (list[str]): Paths to reorganized trace files.\n"
    "    index_dir (str): Directory containing `.dftindex` stores.\n",
    /* tp_doc */
    0,                                    /* tp_traverse */
    0,                                    /* tp_clear */
    0,                                    /* tp_richcompare */
    0,                                    /* tp_weaklistoffset */
    0,                                    /* tp_iter */
    0,                                    /* tp_iternext */
    ReconstructionPlanner_methods,        /* tp_methods */
    0,                                    /* tp_members */
    0,                                    /* tp_getset */
    0,                                    /* tp_base */
    0,                                    /* tp_dict */
    0,                                    /* tp_descr_get */
    0,                                    /* tp_descr_set */
    0,                                    /* tp_dictoffset */
    (initproc)ReconstructionPlanner_init, /* tp_init */
    0,                                    /* tp_alloc */
    ReconstructionPlanner_new,            /* tp_new */
};

int init_reconstruction_planner(PyObject *m) {
    if (PyType_Ready(&ReconstructionPlannerType) < 0) return -1;

    Py_INCREF(&ReconstructionPlannerType);
    if (PyModule_AddObject(m, "ReconstructionPlannerUtility",
                           (PyObject *)&ReconstructionPlannerType) < 0) {
        Py_DECREF(&ReconstructionPlannerType);
        Py_DECREF(m);
        return -1;
    }

    return 0;
}

1	#include <dftracer/utils/core/runtime.h>
2	#include <dftracer/utils/python/py_dict_helpers.h>
3	#include <dftracer/utils/python/runtime.h>
4	#include <dftracer/utils/python/utilities/reconstruction_planner.h>
5	#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>
6
7	#include <string>
8	#include <vector>
9
10	using dftracer::utils::Runtime;
11	using namespace dftracer::utils::utilities::composites::dft::reorganize;
12
13	static Runtime get_runtime(ReconstructionPlannerObject self) {	4✔
14	if (self->runtime_obj)	4!
15	return ((RuntimeObject *)self->runtime_obj)->runtime.get();	×
16	return get_default_runtime();	4✔
17	}	2✔
18
19	static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {	4✔
20	Py_XDECREF(self->runtime_obj);	4✔
21	Py_TYPE(self)->tp_free((PyObject *)self);	4✔
22	}	4✔
23
24	static PyObject ReconstructionPlanner_new(PyTypeObject type, PyObject *args,	4✔
25	PyObject *kwds) {
26	ReconstructionPlannerObject *self;
27	self = (ReconstructionPlannerObject *)type->tp_alloc(type, 0);	4✔
28	if (self != NULL) {	4✔
29	self->runtime_obj = NULL;	4✔
30	}	2✔
31	return (PyObject *)self;	4✔
32	}
33
34	static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,	4✔
35	PyObject args, PyObject kwds) {
36	static const char *kwlist[] = {"runtime", NULL};
37	PyObject *runtime_arg = NULL;	4✔
38
39	if (!PyArg_ParseTupleAndKeywords(args, kwds, "\|O", (char **)kwlist,	4!
40	&runtime_arg)) {
41	return -1;	×
42	}
43
44	if (runtime_arg && runtime_arg != Py_None) {	4!
45	if (PyObject_TypeCheck(runtime_arg, &RuntimeType)) {	×
46	Py_INCREF(runtime_arg);	×
47	self->runtime_obj = runtime_arg;	×
48	} else {
49	PyObject *native = PyObject_GetAttrString(runtime_arg, "_native");	×
50	if (native && PyObject_TypeCheck(native, &RuntimeType)) {	×
51	self->runtime_obj = native;	×
52	} else {
53	Py_XDECREF(native);	×
54	PyErr_SetString(PyExc_TypeError,	×
55	"runtime must be a Runtime instance or None");
56	return -1;	×
57	}
58	}
59	}
60
61	return 0;	4✔
62	}	2✔
63
64	static PyObject ReconstructionPlanner_plan(ReconstructionPlannerObject self,	4✔
65	PyObject args, PyObject kwds) {
66	using dftracer::utils::coro::CoroTask;
67
68	static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
69	PyObject *files_obj;
70	const char *index_dir = "";	4✔
71
72	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O\|s", (char **)kwlist,	4!
73	&files_obj, &index_dir))
74	return NULL;	×
75
76	if (!PyList_Check(files_obj)) {	4✔
77	PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");	×
78	return NULL;	×
79	}
80
81	Py_ssize_t nfiles = PyList_Size(files_obj);	4!
82	std::vector<std::string> files;	4✔
83	files.reserve(static_cast<std::size_t>(nfiles));	4!
84	for (Py_ssize_t i = 0; i < nfiles; i++) {	4!
85	PyObject *item = PyList_GetItem(files_obj, i);	×
86	const char *s = PyUnicode_AsUTF8(item);	×
87	if (!s) return NULL;	×
88	files.emplace_back(s);	×
89	}
90
91	ReconstructionPlannerInput input;	4✔
92	input.reorganized_files = std::move(files);	4✔
93	input.index_dir = index_dir;	4!
94
95	ReconstructionPlan plan;	4✔
96	auto *plan_p = &plan;	4✔
97	ReconstructionPlannerInput input_copy = input;	4!
98	std::string error_msg;	4✔
99
100	Py_BEGIN_ALLOW_THREADS try {	4!
101	Runtime *rt = get_runtime(self);	4!
102	auto task = [plan_p, input_copy]() -> CoroTask<void> {	18!
103	ReconstructionPlannerUtility util;	6!
104	*plan_p = co_await util.process(input_copy);	8!
105	};	12!
106	rt->submit(task(), "reconstruction-planner").get();	4!
107	} catch (const std::exception &e) {	4!
108	error_msg = e.what();	×
109	}	×
110	Py_END_ALLOW_THREADS	4!
111
112	if (!error_msg.empty()) {	4!
113	PyErr_SetString(PyExc_RuntimeError, error_msg.c_str());	×
114	return NULL;	×
115	}
116
117	// files dict: original_path -> reconstruction info
118	PyObject *py_files = PyDict_New();	4!
119	if (!py_files) return NULL;	4✔
120
121	for (const auto &[orig_path, recon] : plan.files) {	4!
122	// checkpoint_segments: int -> list of segment dicts
123	PyObject *py_segs = PyDict_New();	×
124	if (!py_segs) {	×
125	Py_DECREF(py_files);	×
126	return NULL;	×
127	}
128
129	for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {	×
130	PyObject *py_seg_list =
131	PyList_New(static_cast<Py_ssize_t>(seg_list.size()));	×
132	if (!py_seg_list) {	×
133	Py_DECREF(py_segs);	×
134	Py_DECREF(py_files);	×
135	return NULL;	×
136	}
137	for (std::size_t si = 0; si < seg_list.size(); si++) {	×
138	const auto &seg = seg_list[si];	×
139	PyObject *sd = PyDict_New();	×
140	if (!sd) {	×
141	Py_DECREF(py_seg_list);	×
142	Py_DECREF(py_segs);	×
143	Py_DECREF(py_files);	×
144	return NULL;	×
145	}
NEW 146	dict_set_steal(sd, "reorg_file",	×
147	PyUnicode_FromString(seg.reorg_file.c_str()));	×
NEW 148	dict_set_steal(sd, "output_line_start",	×
NEW 149	PyLong_FromLong(seg.output_line_start));	×
NEW 150	dict_set_steal(sd, "output_line_end",	×
NEW 151	PyLong_FromLong(seg.output_line_end));	×
NEW 152	dict_set_steal(sd, "source_checkpoint",	×
NEW 153	PyLong_FromLong(seg.source_checkpoint));	×
NEW 154	dict_set_steal(sd, "event_count",	×
NEW 155	PyLong_FromLong(seg.event_count));	×
UNCOV 156	PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);	×
157	}
158	PyObject *py_cp_key = PyLong_FromLong(cp_idx);	×
159	PyDict_SetItem(py_segs, py_cp_key, py_seg_list);	×
160	Py_DECREF(py_cp_key);	×
161	Py_DECREF(py_seg_list);	×
162	}
163
164	PyObject *py_recon = PyDict_New();	×
165	if (!py_recon) {	×
166	Py_DECREF(py_segs);	×
167	Py_DECREF(py_files);	×
168	return NULL;	×
169	}
NEW 170	dict_set_steal(py_recon, "original_path",	×
171	PyUnicode_FromString(recon.original_path.c_str()));	×
NEW 172	dict_set_steal(py_recon, "num_checkpoints",	×
NEW 173	PyLong_FromLong(recon.num_checkpoints));	×
NEW 174	dict_set_steal(py_recon, "event_hash",	×
175	PyUnicode_FromString(recon.event_hash.c_str()));	×
UNCOV 176	PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);	×
177	Py_DECREF(py_segs);	×
178
179	PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);	×
180	Py_DECREF(py_recon);	×
181	}
182
183	PyObject *result = PyDict_New();	4!
184	if (!result) {	4!
185	Py_DECREF(py_files);	×
186	return NULL;	×
187	}
188	PyDict_SetItemString(result, "files", py_files);	4!
189	Py_DECREF(py_files);	2!
190	dict_set_steal(result, "total_segments",	4!
191	PyLong_FromSize_t(plan.total_segments));	2!
192	dict_set_steal(result, "total_events",	4!
193	PyLong_FromSize_t(plan.total_events));	2!
194	return result;	4✔
195	}	4✔
196
197	static PyObject ReconstructionPlanner_call(PyObject self, PyObject *args,	2✔
198	PyObject *kwds) {
199	return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,	3✔
200	kwds);	2✔
201	}
202
203	static PyMethodDef ReconstructionPlanner_methods[] = {
204	{"process", (PyCFunction)ReconstructionPlanner_plan,
205	METH_VARARGS \| METH_KEYWORDS,
206	"process(reorganized_files, index_dir='')\n"
207	"--\n"
208	"\n"
209	"Build a reconstruction plan from reorganized files.\n"
210	"\n"
211	"Args:\n"
212	" reorganized_files (list[str]): Paths to reorganized files.\n"
213	" index_dir (str): Directory for .dftindex stores (default '').\n"
214	"\n"
215	"Returns:\n"
216	" dict: Reconstruction plan.\n"},
217	{NULL} /* Sentinel */
218	};
219
220	PyTypeObject ReconstructionPlannerType = {
221	PyVarObject_HEAD_INIT(
222	NULL,
223	0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
224	sizeof(ReconstructionPlannerObject), /* tp_basicsize */
225	0, /* tp_itemsize */
226	(destructor)ReconstructionPlanner_dealloc, /* tp_dealloc */
227	0, /* tp_vectorcall_offset */
228	0, /* tp_getattr */
229	0, /* tp_setattr */
230	0, /* tp_as_async */
231	0, /* tp_repr */
232	0, /* tp_as_number */
233	0, /* tp_as_sequence */
234	0, /* tp_as_mapping */
235	0, /* tp_hash */
236	ReconstructionPlanner_call, /* tp_call */
237	0, /* tp_str */
238	0, /* tp_getattro */
239	0, /* tp_setattro */
240	0, /* tp_as_buffer */
241	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
242	"ReconstructionPlannerUtility(runtime: Runtime \| None = None)\n"
243	"--\n"
244	"\n"
245	"Plan reconstruction of original files from reorganized trace files.\n"
246	"\n"
247	"Args:\n"
248	" runtime (Runtime or None): Runtime for thread pool control.\n"
249	" If None, uses the default global Runtime.\n"
250	"\n"
251	"process(reorganized_files, index_dir='') -> dict\n"
252	" reorganized_files (list[str]): Paths to reorganized trace files.\n"
253	" index_dir (str): Directory containing `.dftindex` stores.\n",
254	/* tp_doc */
255	0, /* tp_traverse */
256	0, /* tp_clear */
257	0, /* tp_richcompare */
258	0, /* tp_weaklistoffset */
259	0, /* tp_iter */
260	0, /* tp_iternext */
261	ReconstructionPlanner_methods, /* tp_methods */
262	0, /* tp_members */
263	0, /* tp_getset */
264	0, /* tp_base */
265	0, /* tp_dict */
266	0, /* tp_descr_get */
267	0, /* tp_descr_set */
268	0, /* tp_dictoffset */
269	(initproc)ReconstructionPlanner_init, /* tp_init */
270	0, /* tp_alloc */
271	ReconstructionPlanner_new, /* tp_new */
272	};
273
274	int init_reconstruction_planner(PyObject *m) {	2✔
275	if (PyType_Ready(&ReconstructionPlannerType) < 0) return -1;	2✔
276
277	Py_INCREF(&ReconstructionPlannerType);	1✔
278	if (PyModule_AddObject(m, "ReconstructionPlannerUtility",	3!
279	(PyObject *)&ReconstructionPlannerType) < 0) {	2!
280	Py_DECREF(&ReconstructionPlannerType);
281	Py_DECREF(m);
282	return -1;	×
283	}
284
285	return 0;	2✔
286	}	1✔

llnl / dftracer-utils / 27739884676

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous