28693295402

Committed 04 Jul 2026 03:17AM UTC coverage: 52.408% (+0.1%) from 52.278%

Build # 28693295402

Build Type

push

github

Committed by

hariharan-devarajan

Commit Message

feat: silence noisy warnings on aarch64

Coverage Stats

37318 of 92666 branches covered (40.27%)

Branch coverage included in aggregate %.

33462 of 42389 relevant lines covered (78.94%)

20557.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.5

/src/dftracer/utils/python/utilities/reorganization_planner.cpp

#include <dftracer/utils/core/runtime.h>
#include <dftracer/utils/core/tasks/coro_scope.h>
#include <dftracer/utils/core/utilities/utility_executor.h>
#include <dftracer/utils/python/py_dict_helpers.h>
#include <dftracer/utils/python/py_list_helpers.h>
#include <dftracer/utils/python/py_runtime_mixin.h>
#include <dftracer/utils/python/py_type_helpers.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/utilities/reorganization_planner.h>
#include <dftracer/utils/utilities/composites/dft/reorganize/reorganization_planner.h>

#include <string>
#include <vector>

using dftracer::utils::CoroScope;
using dftracer::utils::Runtime;
using dftracer::utils::utilities::behaviors::UtilityExecutor;
namespace tags = dftracer::utils::utilities::tags;
using namespace dftracer::utils::utilities::composites::dft::reorganize;

DFTRACER_UTILS_RUNTIME_BACKED_SLOTS(ReorganizationPlanner,
                                    ReorganizationPlannerObject)

static PyObject *ReorganizationPlanner_plan(ReorganizationPlannerObject *self,
                                            PyObject *args, PyObject *kwds) {
    using dftracer::utils::coro::CoroTask;

    static const char *kwlist[] = {"source_files", "groups", "index_dir", NULL};
    PyObject *source_files_obj;
    PyObject *groups_obj = Py_None;
    const char *index_dir = "";

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Os", (char **)kwlist,
                                     &source_files_obj, &groups_obj,
                                     &index_dir))
        return NULL;

    std::vector<std::string> files;
    if (!parse_str_list(source_files_obj, "source_files", files)) return NULL;

    std::vector<PredicateGroup> groups;
    if (groups_obj && groups_obj != Py_None) {
        if (!PyList_Check(groups_obj)) {
            PyErr_SetString(PyExc_TypeError,
                            "groups must be a list of dicts or None");
            return NULL;
        }
        Py_ssize_t n = PyList_Size(groups_obj);
        groups.reserve(static_cast<std::size_t>(n));
        for (Py_ssize_t i = 0; i < n; i++) {
            PyObject *item = PyList_GetItem(groups_obj, i);
            PredicateGroup g;
            PyObject *name = PyDict_GetItemString(item, "name");
            PyObject *pred = PyDict_GetItemString(item, "query");
            if (name) {
                const char *ns = PyUnicode_AsUTF8(name);
                if (!ns) return NULL;
                g.name = ns;
            }
            if (pred) {
                const char *ps = PyUnicode_AsUTF8(pred);
                if (!ps) return NULL;
                g.query = ps;
            }
            groups.push_back(std::move(g));
        }
    }

    ReorganizationPlannerInput input;
    input.source_files = std::move(files);
    input.groups = std::move(groups);
    input.index_dir = index_dir;

    ExtractionPlan plan;
    auto *plan_p = &plan;
    ReorganizationPlannerInput input_copy = input;

    if (!run_blocking([&] {
            Runtime *rt = resolve_runtime(self);
            auto task = run_coro_scope(
                rt->executor(),
                [plan_p, input_copy](CoroScope &scope) -> CoroTask<void> {
                    auto planner =
                        std::make_shared<ReorganizationPlannerUtility>();
                    UtilityExecutor<ReorganizationPlannerInput, ExtractionPlan,
                                    tags::NeedsContext>
                        exec(planner);
                    *plan_p = co_await exec.execute(scope, input_copy);
                });
            rt->submit(std::move(task), "reorganization-planner").wait();
        })) {
        return NULL;
    }

    // groups list
    PyObject *py_groups =
        PyList_New(static_cast<Py_ssize_t>(plan.groups.size()));
    if (!py_groups) return NULL;
    for (std::size_t i = 0; i < plan.groups.size(); i++) {
        PyObject *g = PyDict_New();
        if (!g) {
            Py_DECREF(py_groups);
            return NULL;
        }
        dict_set_steal(g, "name",
                       PyUnicode_FromString(plan.groups[i].name.c_str()));
        dict_set_steal(g, "query",
                       PyUnicode_FromString(plan.groups[i].query.c_str()));
        PyList_SetItem(py_groups, static_cast<Py_ssize_t>(i), g);
    }

    // source_files list
    PyObject *py_sources =
        PyList_New(static_cast<Py_ssize_t>(plan.source_files.size()));
    if (!py_sources) {
        Py_DECREF(py_groups);
        return NULL;
    }
    for (std::size_t i = 0; i < plan.source_files.size(); i++) {
        const auto &sf = plan.source_files[i];
        PyObject *entry = PyDict_New();
        if (!entry) {
            Py_DECREF(py_groups);
            Py_DECREF(py_sources);
            return NULL;
        }
        dict_set_steal(entry, "file_path",
                       PyUnicode_FromString(sf.file_path.c_str()));
        dict_set_steal(entry, "index_path",
                       PyUnicode_FromString(sf.index_path.c_str()));
        dict_set_steal(entry, "num_checkpoints",
                       PyLong_FromSize_t(sf.num_checkpoints));
        dict_set_steal(entry, "uncompressed_size",
                       PyLong_FromUnsignedLongLong(sf.uncompressed_size));
        dict_set_steal(entry, "checkpoint_size",
                       PyLong_FromUnsignedLongLong(sf.checkpoint_size));
        PyList_SetItem(py_sources, static_cast<Py_ssize_t>(i), entry);
    }

    // tasks list
    PyObject *py_tasks = PyList_New(static_cast<Py_ssize_t>(plan.tasks.size()));
    if (!py_tasks) {
        Py_DECREF(py_groups);
        Py_DECREF(py_sources);
        return NULL;
    }
    for (std::size_t i = 0; i < plan.tasks.size(); i++) {
        const auto &t = plan.tasks[i];
        PyObject *entry = PyDict_New();
        if (!entry) {
            Py_DECREF(py_groups);
            Py_DECREF(py_sources);
            Py_DECREF(py_tasks);
            return NULL;
        }
        dict_set_steal(entry, "source_file_idx",
                       PyLong_FromSize_t(t.source_file_idx));
        dict_set_steal(entry, "checkpoint_idx",
                       PyLong_FromUnsignedLongLong(t.checkpoint_idx));
        dict_set_steal(entry, "target_group",
                       PyUnicode_FromString(t.target_group.c_str()));
        dict_set_steal(entry, "start_byte",
                       PyLong_FromUnsignedLongLong(t.start_byte));
        dict_set_steal(entry, "end_byte",
                       PyLong_FromUnsignedLongLong(t.end_byte));
        PyList_SetItem(py_tasks, static_cast<Py_ssize_t>(i), entry);
    }

    PyObject *result = PyDict_New();
    if (!result) {
        Py_DECREF(py_groups);
        Py_DECREF(py_sources);
        Py_DECREF(py_tasks);
        return NULL;
    }
    PyDict_SetItemString(result, "groups", py_groups);
    Py_DECREF(py_groups);
    PyDict_SetItemString(result, "source_files", py_sources);
    Py_DECREF(py_sources);
    PyDict_SetItemString(result, "tasks", py_tasks);
    Py_DECREF(py_tasks);
    dict_set_steal(result, "total_events",
                   PyLong_FromSize_t(plan.total_events));
    return result;
}

static PyObject *ReorganizationPlanner_call(PyObject *self, PyObject *args,
                                            PyObject *kwds) {
    return ReorganizationPlanner_plan((ReorganizationPlannerObject *)self, args,
                                      kwds);
}

static PyMethodDef ReorganizationPlanner_methods[] = {
    {"process", (PyCFunction)ReorganizationPlanner_plan,
     METH_VARARGS | METH_KEYWORDS,
     "process(source_files, groups=None, index_dir='')\n"
     "--\n"
     "\n"
     "Build a reorganization plan for trace files.\n"
     "\n"
     "Args:\n"
     "    source_files (list[str]): Paths to source trace files.\n"
     "    groups (list[dict] or None): Predicate group definitions\n"
     "        (default None).\n"
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
     "\n"
     "Returns:\n"
     "    dict: Extraction plan.\n"},
    {NULL} /* Sentinel */
};

PyTypeObject ReorganizationPlannerType = {
    PyVarObject_HEAD_INIT(
        NULL,
        0) "dftracer_utils_ext.ReorganizationPlannerUtility", /* tp_name */
    sizeof(ReorganizationPlannerObject),                      /* tp_basicsize */
    0,                                                        /* tp_itemsize */
    (destructor)ReorganizationPlanner_dealloc,                /* tp_dealloc */
    0,                                        /* tp_vectorcall_offset */
    0,                                        /* tp_getattr */
    0,                                        /* tp_setattr */
    0,                                        /* tp_as_async */
    0,                                        /* tp_repr */
    0,                                        /* tp_as_number */
    0,                                        /* tp_as_sequence */
    0,                                        /* tp_as_mapping */
    0,                                        /* tp_hash */
    ReorganizationPlanner_call,               /* tp_call */
    0,                                        /* tp_str */
    0,                                        /* tp_getattro */
    0,                                        /* tp_setattro */
    0,                                        /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
    "ReorganizationPlannerUtility(runtime: Runtime | None = None)\n"
    "--\n"
    "\n"
    "Plan semantic reorganization of trace files into predicate groups.\n"
    "\n"
    "Args:\n"
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
    "        If None, uses the default global Runtime.\n", /* tp_doc */
    0,                                                     /* tp_traverse */
    0,                                                     /* tp_clear */
    0,                                                     /* tp_richcompare */
    0,                                    /* tp_weaklistoffset */
    0,                                    /* tp_iter */
    0,                                    /* tp_iternext */
    ReorganizationPlanner_methods,        /* tp_methods */
    0,                                    /* tp_members */
    0,                                    /* tp_getset */
    0,                                    /* tp_base */
    0,                                    /* tp_dict */
    0,                                    /* tp_descr_get */
    0,                                    /* tp_descr_set */
    0,                                    /* tp_dictoffset */
    (initproc)ReorganizationPlanner_init, /* tp_init */
    0,                                    /* tp_alloc */
    ReorganizationPlanner_new,            /* tp_new */
};

int init_reorganization_planner(PyObject *m) {
    if (register_type(m, &ReorganizationPlannerType,
                      "ReorganizationPlannerUtility") < 0)
        return -1;

    return 0;
}

1	#include <dftracer/utils/core/runtime.h>
2	#include <dftracer/utils/core/tasks/coro_scope.h>
3	#include <dftracer/utils/core/utilities/utility_executor.h>
4	#include <dftracer/utils/python/py_dict_helpers.h>
5	#include <dftracer/utils/python/py_list_helpers.h>
6	#include <dftracer/utils/python/py_runtime_mixin.h>
7	#include <dftracer/utils/python/py_type_helpers.h>
8	#include <dftracer/utils/python/runtime.h>
9	#include <dftracer/utils/python/utilities/reorganization_planner.h>
10	#include <dftracer/utils/utilities/composites/dft/reorganize/reorganization_planner.h>
11
12	#include <string>
13	#include <vector>
14
15	using dftracer::utils::CoroScope;
16	using dftracer::utils::Runtime;
17	using dftracer::utils::utilities::behaviors::UtilityExecutor;
18	namespace tags = dftracer::utils::utilities::tags;
19	using namespace dftracer::utils::utilities::composites::dft::reorganize;
20
21	DFTRACER_UTILS_RUNTIME_BACKED_SLOTS(ReorganizationPlanner,	24✔
22	ReorganizationPlannerObject)
23
24	static PyObject ReorganizationPlanner_plan(ReorganizationPlannerObject self,	8✔
25	PyObject args, PyObject kwds) {
26	using dftracer::utils::coro::CoroTask;
27
28	static const char *kwlist[] = {"source_files", "groups", "index_dir", NULL};
29	PyObject *source_files_obj;
30	PyObject *groups_obj = Py_None;	8✔
31	const char *index_dir = "";	8✔
32
33	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O\|Os", (char **)kwlist,	8!
34	&source_files_obj, &groups_obj,
35	&index_dir))
36	return NULL;	×
37
38	std::vector<std::string> files;	8✔
39	if (!parse_str_list(source_files_obj, "source_files", files)) return NULL;	8!
40
41	std::vector<PredicateGroup> groups;	8✔
42	if (groups_obj && groups_obj != Py_None) {	8!
43	if (!PyList_Check(groups_obj)) {	8!
44	PyErr_SetString(PyExc_TypeError,	×
45	"groups must be a list of dicts or None");
46	return NULL;	×
47	}
48	Py_ssize_t n = PyList_Size(groups_obj);	8!
49	groups.reserve(static_cast<std::size_t>(n));	8!
50	for (Py_ssize_t i = 0; i < n; i++) {	16✔
51	PyObject *item = PyList_GetItem(groups_obj, i);	8!
52	PredicateGroup g;	8✔
53	PyObject *name = PyDict_GetItemString(item, "name");	8!
54	PyObject *pred = PyDict_GetItemString(item, "query");	8!
55	if (name) {	8!
56	const char *ns = PyUnicode_AsUTF8(name);	8!
57	if (!ns) return NULL;	8✔
58	g.name = ns;	8!
59	}	4✔
60	if (pred) {	8!
61	const char *ps = PyUnicode_AsUTF8(pred);	8!
62	if (!ps) return NULL;	8✔
63	g.query = ps;	8!
64	}	4✔
65	groups.push_back(std::move(g));	8!
66	}	8✔
67	}	4✔
68
69	ReorganizationPlannerInput input;	8✔
70	input.source_files = std::move(files);	8✔
71	input.groups = std::move(groups);	8✔
72	input.index_dir = index_dir;	8!
73
74	ExtractionPlan plan;	8✔
75	auto *plan_p = &plan;	8✔
76	ReorganizationPlannerInput input_copy = input;	8!
77
78	if (!run_blocking([&] {	12!
79	Runtime *rt = resolve_runtime(self);	8!
80	auto task = run_coro_scope(	4!
81	rt->executor(),	4✔
82	[plan_p, input_copy](CoroScope &scope) -> CoroTask<void> {	36!
83	auto planner =	12✔
84	std::make_shared<ReorganizationPlannerUtility>();	12!
85	UtilityExecutor<ReorganizationPlannerInput, ExtractionPlan,	12✔
86	tags::NeedsContext>
87	exec(planner);	12!
88	*plan_p = co_await exec.execute(scope, input_copy);	20!
89	});	24!
90	rt->submit(std::move(task), "reorganization-planner").wait();	8!
91	})) {	8✔
92	return NULL;	×
93	}
94
95	// groups list
96	PyObject *py_groups =	4✔
97	PyList_New(static_cast<Py_ssize_t>(plan.groups.size()));	8!
98	if (!py_groups) return NULL;	8✔
99	for (std::size_t i = 0; i < plan.groups.size(); i++) {	24✔
100	PyObject *g = PyDict_New();	16!
101	if (!g) {	16!
102	Py_DECREF(py_groups);	×
103	return NULL;	×
104	}
105	dict_set_steal(g, "name",	16!
106	PyUnicode_FromString(plan.groups[i].name.c_str()));	16!
107	dict_set_steal(g, "query",	16!
108	PyUnicode_FromString(plan.groups[i].query.c_str()));	16!
109	PyList_SetItem(py_groups, static_cast<Py_ssize_t>(i), g);	16!
110	}	8✔
111
112	// source_files list
113	PyObject *py_sources =	4✔
114	PyList_New(static_cast<Py_ssize_t>(plan.source_files.size()));	8!
115	if (!py_sources) {	8!
116	Py_DECREF(py_groups);	×
117	return NULL;	×
118	}
119	for (std::size_t i = 0; i < plan.source_files.size(); i++) {	16✔
120	const auto &sf = plan.source_files[i];	8✔
121	PyObject *entry = PyDict_New();	8!
122	if (!entry) {	8!
123	Py_DECREF(py_groups);	×
124	Py_DECREF(py_sources);	×
125	return NULL;	×
126	}
127	dict_set_steal(entry, "file_path",	8!
128	PyUnicode_FromString(sf.file_path.c_str()));	4!
129	dict_set_steal(entry, "index_path",	8!
130	PyUnicode_FromString(sf.index_path.c_str()));	4!
131	dict_set_steal(entry, "num_checkpoints",	8!
132	PyLong_FromSize_t(sf.num_checkpoints));	8!
133	dict_set_steal(entry, "uncompressed_size",	8!
134	PyLong_FromUnsignedLongLong(sf.uncompressed_size));	8!
135	dict_set_steal(entry, "checkpoint_size",	8!
136	PyLong_FromUnsignedLongLong(sf.checkpoint_size));	8!
137	PyList_SetItem(py_sources, static_cast<Py_ssize_t>(i), entry);	8!
138	}	4✔
139
140	// tasks list
141	PyObject *py_tasks = PyList_New(static_cast<Py_ssize_t>(plan.tasks.size()));	8!
142	if (!py_tasks) {	8!
143	Py_DECREF(py_groups);	×
144	Py_DECREF(py_sources);	×
145	return NULL;	×
146	}
147	for (std::size_t i = 0; i < plan.tasks.size(); i++) {	20✔
148	const auto &t = plan.tasks[i];	12✔
149	PyObject *entry = PyDict_New();	12!
150	if (!entry) {	12!
151	Py_DECREF(py_groups);	×
152	Py_DECREF(py_sources);	×
153	Py_DECREF(py_tasks);	×
154	return NULL;	×
155	}
156	dict_set_steal(entry, "source_file_idx",	12!
157	PyLong_FromSize_t(t.source_file_idx));	12!
158	dict_set_steal(entry, "checkpoint_idx",	12!
159	PyLong_FromUnsignedLongLong(t.checkpoint_idx));	12!
160	dict_set_steal(entry, "target_group",	12!
161	PyUnicode_FromString(t.target_group.c_str()));	6!
162	dict_set_steal(entry, "start_byte",	12!
163	PyLong_FromUnsignedLongLong(t.start_byte));	12!
164	dict_set_steal(entry, "end_byte",	12!
165	PyLong_FromUnsignedLongLong(t.end_byte));	12!
166	PyList_SetItem(py_tasks, static_cast<Py_ssize_t>(i), entry);	12!
167	}	6✔
168
169	PyObject *result = PyDict_New();	8!
170	if (!result) {	8!
171	Py_DECREF(py_groups);	×
172	Py_DECREF(py_sources);	×
173	Py_DECREF(py_tasks);	×
174	return NULL;	×
175	}
176	PyDict_SetItemString(result, "groups", py_groups);	8!
177	Py_DECREF(py_groups);	4!
178	PyDict_SetItemString(result, "source_files", py_sources);	8!
179	Py_DECREF(py_sources);	4!
180	PyDict_SetItemString(result, "tasks", py_tasks);	8!
181	Py_DECREF(py_tasks);	4!
182	dict_set_steal(result, "total_events",	8!
183	PyLong_FromSize_t(plan.total_events));	4!
184	return result;	8✔
185	}	8✔
186
187	static PyObject ReorganizationPlanner_call(PyObject self, PyObject *args,	2✔
188	PyObject *kwds) {
189	return ReorganizationPlanner_plan((ReorganizationPlannerObject *)self, args,	3✔
190	kwds);	2✔
191	}
192
193	static PyMethodDef ReorganizationPlanner_methods[] = {
194	{"process", (PyCFunction)ReorganizationPlanner_plan,
195	METH_VARARGS \| METH_KEYWORDS,
196	"process(source_files, groups=None, index_dir='')\n"
197	"--\n"
198	"\n"
199	"Build a reorganization plan for trace files.\n"
200	"\n"
201	"Args:\n"
202	" source_files (list[str]): Paths to source trace files.\n"
203	" groups (list[dict] or None): Predicate group definitions\n"
204	" (default None).\n"
205	" index_dir (str): Directory for .dftindex stores (default '').\n"
206	"\n"
207	"Returns:\n"
208	" dict: Extraction plan.\n"},
209	{NULL} /* Sentinel */
210	};
211
212	PyTypeObject ReorganizationPlannerType = {
213	PyVarObject_HEAD_INIT(
214	NULL,
215	0) "dftracer_utils_ext.ReorganizationPlannerUtility", /* tp_name */
216	sizeof(ReorganizationPlannerObject), /* tp_basicsize */
217	0, /* tp_itemsize */
218	(destructor)ReorganizationPlanner_dealloc, /* tp_dealloc */
219	0, /* tp_vectorcall_offset */
220	0, /* tp_getattr */
221	0, /* tp_setattr */
222	0, /* tp_as_async */
223	0, /* tp_repr */
224	0, /* tp_as_number */
225	0, /* tp_as_sequence */
226	0, /* tp_as_mapping */
227	0, /* tp_hash */
228	ReorganizationPlanner_call, /* tp_call */
229	0, /* tp_str */
230	0, /* tp_getattro */
231	0, /* tp_setattro */
232	0, /* tp_as_buffer */
233	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
234	"ReorganizationPlannerUtility(runtime: Runtime \| None = None)\n"
235	"--\n"
236	"\n"
237	"Plan semantic reorganization of trace files into predicate groups.\n"
238	"\n"
239	"Args:\n"
240	" runtime (Runtime or None): Runtime for thread pool control.\n"
241	" If None, uses the default global Runtime.\n", /* tp_doc */
242	0, /* tp_traverse */
243	0, /* tp_clear */
244	0, /* tp_richcompare */
245	0, /* tp_weaklistoffset */
246	0, /* tp_iter */
247	0, /* tp_iternext */
248	ReorganizationPlanner_methods, /* tp_methods */
249	0, /* tp_members */
250	0, /* tp_getset */
251	0, /* tp_base */
252	0, /* tp_dict */
253	0, /* tp_descr_get */
254	0, /* tp_descr_set */
255	0, /* tp_dictoffset */
256	(initproc)ReorganizationPlanner_init, /* tp_init */
257	0, /* tp_alloc */
258	ReorganizationPlanner_new, /* tp_new */
259	};
260
261	int init_reorganization_planner(PyObject *m) {	2✔
262	if (register_type(m, &ReorganizationPlannerType,	2✔
263	"ReorganizationPlannerUtility") < 0)	2✔
264	return -1;	×
265
266	return 0;	2✔
267	}	1✔

llnl / dftracer-utils / 28693295402

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous