28398085302

Committed 29 Jun 2026 07:43PM UTC coverage: 50.067% (-2.2%) from 52.278%

Build # 28398085302

Build Type

Pull #83

github

Committed by

web-flow

Commit Message

Merge 9a615085e into 2efed6649

Pull Request Pull Request #83: refactor and improve code QoL

Coverage Stats

16342 of 44293 branches covered (36.9%)

Branch coverage included in aggregate %.

613 of 1132 new or added lines in 52 files covered. (54.15%)

687 existing lines in 116 files now uncovered.

21698 of 31685 relevant lines covered (68.48%)

12958.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33

/src/dftracer/utils/python/utilities/reconstruction_planner.cpp

#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/python/py_dict_helpers.h>
#include <dftracer/utils/python/py_runtime_mixin.h>
#include <dftracer/utils/python/py_type_helpers.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/reconstruction_planner.h>
#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>

#include <string>
#include <vector>

using dftracer::utils::Runtime;
using namespace dftracer::utils::utilities::composites::dft::reorganize;

static Runtime *get_runtime(ReconstructionPlannerObject *self) {
    return resolve_runtime(self);
}

static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {
    runtime_backed_dealloc(self);
}

static PyObject *ReconstructionPlanner_new(PyTypeObject *type, PyObject *args,
                                           PyObject *kwds) {
    return runtime_backed_new<ReconstructionPlannerObject>(type, args, kwds);
}

static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,
                                      PyObject *args, PyObject *kwds) {
    return runtime_backed_init(self, args, kwds);
}

static PyObject *ReconstructionPlanner_plan(ReconstructionPlannerObject *self,
                                            PyObject *args, PyObject *kwds) {
    using dftracer::utils::coro::CoroTask;

    static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
    PyObject *files_obj;
    const char *index_dir = "";

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s", (char **)kwlist,
                                     &files_obj, &index_dir))
        return NULL;

    if (!PyList_Check(files_obj)) {
        PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");
        return NULL;
    }

    Py_ssize_t nfiles = PyList_Size(files_obj);
    std::vector<std::string> files;
    files.reserve(static_cast<std::size_t>(nfiles));
    for (Py_ssize_t i = 0; i < nfiles; i++) {
        PyObject *item = PyList_GetItem(files_obj, i);
        const char *s = PyUnicode_AsUTF8(item);
        if (!s) return NULL;
        files.emplace_back(s);
    }

    ReconstructionPlannerInput input;
    input.reorganized_files = std::move(files);
    input.index_dir = index_dir;

    ReconstructionPlan plan;
    auto *plan_p = &plan;
    ReconstructionPlannerInput input_copy = input;

    if (!run_blocking([&] {
            Runtime *rt = get_runtime(self);
            auto task = [plan_p, input_copy]() -> CoroTask<void> {
                ReconstructionPlannerUtility util;
                *plan_p = co_await util.process(input_copy);
            };
            rt->submit(task(), "reconstruction-planner").get();
        })) {
        return NULL;
    }

    // files dict: original_path -> reconstruction info
    PyObject *py_files = PyDict_New();
    if (!py_files) return NULL;

    for (const auto &[orig_path, recon] : plan.files) {
        // checkpoint_segments: int -> list of segment dicts
        PyObject *py_segs = PyDict_New();
        if (!py_segs) {
            Py_DECREF(py_files);
            return NULL;
        }

        for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {
            PyObject *py_seg_list =
                PyList_New(static_cast<Py_ssize_t>(seg_list.size()));
            if (!py_seg_list) {
                Py_DECREF(py_segs);
                Py_DECREF(py_files);
                return NULL;
            }
            for (std::size_t si = 0; si < seg_list.size(); si++) {
                const auto &seg = seg_list[si];
                PyObject *sd = PyDict_New();
                if (!sd) {
                    Py_DECREF(py_seg_list);
                    Py_DECREF(py_segs);
                    Py_DECREF(py_files);
                    return NULL;
                }
                dict_set_steal(sd, "reorg_file",
                               PyUnicode_FromString(seg.reorg_file.c_str()));
                dict_set_steal(sd, "output_line_start",
                               PyLong_FromLong(seg.output_line_start));
                dict_set_steal(sd, "output_line_end",
                               PyLong_FromLong(seg.output_line_end));
                dict_set_steal(sd, "source_checkpoint",
                               PyLong_FromLong(seg.source_checkpoint));
                dict_set_steal(sd, "event_count",
                               PyLong_FromLong(seg.event_count));
                PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);
            }
            PyObject *py_cp_key = PyLong_FromLong(cp_idx);
            PyDict_SetItem(py_segs, py_cp_key, py_seg_list);
            Py_DECREF(py_cp_key);
            Py_DECREF(py_seg_list);
        }

        PyObject *py_recon = PyDict_New();
        if (!py_recon) {
            Py_DECREF(py_segs);
            Py_DECREF(py_files);
            return NULL;
        }
        dict_set_steal(py_recon, "original_path",
                       PyUnicode_FromString(recon.original_path.c_str()));
        dict_set_steal(py_recon, "num_checkpoints",
                       PyLong_FromLong(recon.num_checkpoints));
        dict_set_steal(py_recon, "event_hash",
                       PyUnicode_FromString(recon.event_hash.c_str()));
        PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);
        Py_DECREF(py_segs);

        PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);
        Py_DECREF(py_recon);
    }

    PyObject *result = PyDict_New();
    if (!result) {
        Py_DECREF(py_files);
        return NULL;
    }
    PyDict_SetItemString(result, "files", py_files);
    Py_DECREF(py_files);
    dict_set_steal(result, "total_segments",
                   PyLong_FromSize_t(plan.total_segments));
    dict_set_steal(result, "total_events",
                   PyLong_FromSize_t(plan.total_events));
    return result;
}

static PyObject *ReconstructionPlanner_call(PyObject *self, PyObject *args,
                                            PyObject *kwds) {
    return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,
                                      kwds);
}

static PyMethodDef ReconstructionPlanner_methods[] = {
    {"process", (PyCFunction)ReconstructionPlanner_plan,
     METH_VARARGS | METH_KEYWORDS,
     "process(reorganized_files, index_dir='')\n"
     "--\n"
     "\n"
     "Build a reconstruction plan from reorganized files.\n"
     "\n"
     "Args:\n"
     "    reorganized_files (list[str]): Paths to reorganized files.\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Reconstruction plan.\n"},
    {NULL} /* Sentinel */
};

PyTypeObject ReconstructionPlannerType = {
    PyVarObject_HEAD_INIT(
        NULL,
        0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
    sizeof(ReconstructionPlannerObject),                      /* tp_basicsize */
    0,                                                        /* tp_itemsize */
    (destructor)ReconstructionPlanner_dealloc,                /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    ReconstructionPlanner_call,               /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "ReconstructionPlannerUtility(runtime: Runtime | None = None)\n"
    "--\n"
    "\n"
    "Plan reconstruction of original files from reorganized trace files.\n"
    "\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "        If None, uses the default global Runtime.\n"
    "\n"
    "process(reorganized_files, index_dir='') -> dict\n"
    "    reorganized_files (list[str]): Paths to reorganized trace files.\n"
    "    index_dir (str): Directory containing `.dftindex` stores.\n",
    /* tp_doc */
    0,                                    /* tp_traverse */
    0,                                    /* tp_clear */
    0,                                    /* tp_richcompare */
    0,                                    /* tp_weaklistoffset */
    0,                                    /* tp_iter */
    0,                                    /* tp_iternext */
    ReconstructionPlanner_methods,        /* tp_methods */
    0,                                    /* tp_members */
    0,                                    /* tp_getset */
    0,                                    /* tp_base */
    0,                                    /* tp_dict */
    0,                                    /* tp_descr_get */
    0,                                    /* tp_descr_set */
    0,                                    /* tp_dictoffset */
    (initproc)ReconstructionPlanner_init, /* tp_init */
    0,                                    /* tp_alloc */
    ReconstructionPlanner_new,            /* tp_new */
};

int init_reconstruction_planner(PyObject *m) {
    if (register_type(m, &ReconstructionPlannerType,
                      "ReconstructionPlannerUtility") < 0)
        return -1;

    return 0;
}

1	#include <dftracer/utils/core/runtime.h>
2	#include <dftracer/utils/python/py_dict_helpers.h>
3	#include <dftracer/utils/python/py_runtime_mixin.h>
4	#include <dftracer/utils/python/py_type_helpers.h>
5	#include <dftracer/utils/python/runtime.h>
6	#include <dftracer/utils/python/utilities/reconstruction_planner.h>
7	#include <dftracer/utils/utilities/composites/dft/reorganize/reconstruction_planner.h>
8
9	#include <string>
10	#include <vector>
11
12	using dftracer::utils::Runtime;
13	using namespace dftracer::utils::utilities::composites::dft::reorganize;
14
15	static Runtime get_runtime(ReconstructionPlannerObject self) {	2✔
16	return resolve_runtime(self);	2✔
17	}
18
19	static void ReconstructionPlanner_dealloc(ReconstructionPlannerObject *self) {	2✔
20	runtime_backed_dealloc(self);	2✔
21	}	2✔
22
23	static PyObject ReconstructionPlanner_new(PyTypeObject type, PyObject *args,	2✔
24	PyObject *kwds) {
25	return runtime_backed_new<ReconstructionPlannerObject>(type, args, kwds);	2✔
26	}
27
28	static int ReconstructionPlanner_init(ReconstructionPlannerObject *self,	2✔
29	PyObject args, PyObject kwds) {
30	return runtime_backed_init(self, args, kwds);	2✔
31	}
32
33	static PyObject ReconstructionPlanner_plan(ReconstructionPlannerObject self,	2✔
34	PyObject args, PyObject kwds) {
35	using dftracer::utils::coro::CoroTask;
36
37	static const char *kwlist[] = {"reorganized_files", "index_dir", NULL};
38	PyObject *files_obj;
39	const char *index_dir = "";	2✔
40
41	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O\|s", (char **)kwlist,	2!
42	&files_obj, &index_dir))
43	return NULL;	×
44
45	if (!PyList_Check(files_obj)) {	2!
46	PyErr_SetString(PyExc_TypeError, "reorganized_files must be a list");	×
47	return NULL;	×
48	}
49
50	Py_ssize_t nfiles = PyList_Size(files_obj);	2!
51	std::vector<std::string> files;	2✔
52	files.reserve(static_cast<std::size_t>(nfiles));	2!
53	for (Py_ssize_t i = 0; i < nfiles; i++) {	2!
54	PyObject *item = PyList_GetItem(files_obj, i);	×
55	const char *s = PyUnicode_AsUTF8(item);	×
56	if (!s) return NULL;	×
57	files.emplace_back(s);	×
58	}
59
60	ReconstructionPlannerInput input;	2✔
61	input.reorganized_files = std::move(files);	2✔
62	input.index_dir = index_dir;	2!
63
64	ReconstructionPlan plan;	2✔
65	auto *plan_p = &plan;	2✔
66	ReconstructionPlannerInput input_copy = input;	2!
67
68	if (!run_blocking([&] {	2!
69	Runtime *rt = get_runtime(self);	2!
70	auto task = [plan_p, input_copy]() -> CoroTask<void> {	2!
71	ReconstructionPlannerUtility util;
72	*plan_p = co_await util.process(input_copy);
73	};	6!
74	rt->submit(task(), "reconstruction-planner").get();	2!
75	})) {	2✔
UNCOV 76	return NULL;	×
77	}
78
79	// files dict: original_path -> reconstruction info
80	PyObject *py_files = PyDict_New();	2!
81	if (!py_files) return NULL;	2!
82
83	for (const auto &[orig_path, recon] : plan.files) {	2!
84	// checkpoint_segments: int -> list of segment dicts
85	PyObject *py_segs = PyDict_New();	×
86	if (!py_segs) {	×
87	Py_DECREF(py_files);
88	return NULL;	×
89	}
90
91	for (const auto &[cp_idx, seg_list] : recon.checkpoint_segments) {	×
92	PyObject *py_seg_list =
93	PyList_New(static_cast<Py_ssize_t>(seg_list.size()));	×
94	if (!py_seg_list) {	×
95	Py_DECREF(py_segs);
96	Py_DECREF(py_files);
97	return NULL;	×
98	}
99	for (std::size_t si = 0; si < seg_list.size(); si++) {	×
100	const auto &seg = seg_list[si];	×
101	PyObject *sd = PyDict_New();	×
102	if (!sd) {	×
103	Py_DECREF(py_seg_list);
104	Py_DECREF(py_segs);
105	Py_DECREF(py_files);
106	return NULL;	×
107	}
108	dict_set_steal(sd, "reorg_file",	×
109	PyUnicode_FromString(seg.reorg_file.c_str()));
110	dict_set_steal(sd, "output_line_start",	×
111	PyLong_FromLong(seg.output_line_start));	×
112	dict_set_steal(sd, "output_line_end",	×
113	PyLong_FromLong(seg.output_line_end));	×
114	dict_set_steal(sd, "source_checkpoint",	×
115	PyLong_FromLong(seg.source_checkpoint));	×
116	dict_set_steal(sd, "event_count",	×
117	PyLong_FromLong(seg.event_count));	×
118	PyList_SetItem(py_seg_list, static_cast<Py_ssize_t>(si), sd);	×
119	}
120	PyObject *py_cp_key = PyLong_FromLong(cp_idx);	×
121	PyDict_SetItem(py_segs, py_cp_key, py_seg_list);	×
122	Py_DECREF(py_cp_key);
123	Py_DECREF(py_seg_list);
124	}
125
126	PyObject *py_recon = PyDict_New();	×
127	if (!py_recon) {	×
128	Py_DECREF(py_segs);
129	Py_DECREF(py_files);
130	return NULL;	×
131	}
132	dict_set_steal(py_recon, "original_path",	×
133	PyUnicode_FromString(recon.original_path.c_str()));
134	dict_set_steal(py_recon, "num_checkpoints",	×
135	PyLong_FromLong(recon.num_checkpoints));	×
136	dict_set_steal(py_recon, "event_hash",	×
137	PyUnicode_FromString(recon.event_hash.c_str()));
138	PyDict_SetItemString(py_recon, "checkpoint_segments", py_segs);	×
139	Py_DECREF(py_segs);
140
141	PyDict_SetItemString(py_files, orig_path.c_str(), py_recon);	×
142	Py_DECREF(py_recon);
143	}
144
145	PyObject *result = PyDict_New();	2!
146	if (!result) {	2!
147	Py_DECREF(py_files);
148	return NULL;	×
149	}
150	PyDict_SetItemString(result, "files", py_files);	2!
151	Py_DECREF(py_files);
152	dict_set_steal(result, "total_segments",	2!
153	PyLong_FromSize_t(plan.total_segments));
154	dict_set_steal(result, "total_events",	2!
155	PyLong_FromSize_t(plan.total_events));
156	return result;	2✔
157	}	2✔
158
159	static PyObject ReconstructionPlanner_call(PyObject self, PyObject *args,	1✔
160	PyObject *kwds) {
161	return ReconstructionPlanner_plan((ReconstructionPlannerObject *)self, args,	1✔
162	kwds);	1✔
163	}
164
165	static PyMethodDef ReconstructionPlanner_methods[] = {
166	{"process", (PyCFunction)ReconstructionPlanner_plan,
167	METH_VARARGS \| METH_KEYWORDS,
168	"process(reorganized_files, index_dir='')\n"
169	"--\n"
170	"\n"
171	"Build a reconstruction plan from reorganized files.\n"
172	"\n"
173	"Args:\n"
174	" reorganized_files (list[str]): Paths to reorganized files.\n"
175	" index_dir (str): Directory for .dftindex stores (default '').\n"
176	"\n"
177	"Returns:\n"
178	" dict: Reconstruction plan.\n"},
179	{NULL} /* Sentinel */
180	};
181
182	PyTypeObject ReconstructionPlannerType = {
183	PyVarObject_HEAD_INIT(
184	NULL,
185	0) "dftracer_utils_ext.ReconstructionPlannerUtility", /* tp_name */
186	sizeof(ReconstructionPlannerObject), /* tp_basicsize */
187	0, /* tp_itemsize */
188	(destructor)ReconstructionPlanner_dealloc, /* tp_dealloc */
189	0, /* tp_vectorcall_offset */
190	0, /* tp_getattr */
191	0, /* tp_setattr */
192	0, /* tp_as_async */
193	0, /* tp_repr */
194	0, /* tp_as_number */
195	0, /* tp_as_sequence */
196	0, /* tp_as_mapping */
197	0, /* tp_hash */
198	ReconstructionPlanner_call, /* tp_call */
199	0, /* tp_str */
200	0, /* tp_getattro */
201	0, /* tp_setattro */
202	0, /* tp_as_buffer */
203	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
204	"ReconstructionPlannerUtility(runtime: Runtime \| None = None)\n"
205	"--\n"
206	"\n"
207	"Plan reconstruction of original files from reorganized trace files.\n"
208	"\n"
209	"Args:\n"
210	" runtime (Runtime or None): Runtime for thread pool control.\n"
211	" If None, uses the default global Runtime.\n"
212	"\n"
213	"process(reorganized_files, index_dir='') -> dict\n"
214	" reorganized_files (list[str]): Paths to reorganized trace files.\n"
215	" index_dir (str): Directory containing `.dftindex` stores.\n",
216	/* tp_doc */
217	0, /* tp_traverse */
218	0, /* tp_clear */
219	0, /* tp_richcompare */
220	0, /* tp_weaklistoffset */
221	0, /* tp_iter */
222	0, /* tp_iternext */
223	ReconstructionPlanner_methods, /* tp_methods */
224	0, /* tp_members */
225	0, /* tp_getset */
226	0, /* tp_base */
227	0, /* tp_dict */
228	0, /* tp_descr_get */
229	0, /* tp_descr_set */
230	0, /* tp_dictoffset */
231	(initproc)ReconstructionPlanner_init, /* tp_init */
232	0, /* tp_alloc */
233	ReconstructionPlanner_new, /* tp_new */
234	};
235
236	int init_reconstruction_planner(PyObject *m) {	1✔
237	if (register_type(m, &ReconstructionPlannerType,	1✔
238	"ReconstructionPlannerUtility") < 0)	1!
UNCOV 239	return -1;	×
240
241	return 0;	1✔
242	}

llnl / dftracer-utils / 28398085302

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous