• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

llnl / dftracer-utils / 28423703495

30 Jun 2026 05:59AM UTC coverage: 51.998% (-0.3%) from 52.278%
28423703495

Pull #83

github

web-flow
Merge fb542a938 into 2efed6649
Pull Request #83: refactor and improve code QoL

37282 of 93303 branches covered (39.96%)

Branch coverage included in aggregate %.

801 of 1525 new or added lines in 78 files covered. (52.52%)

98 existing lines in 37 files now uncovered.

33674 of 43157 relevant lines covered (78.03%)

20306.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.19
/src/dftracer/utils/python/utilities/reorganization_planner.cpp
1
#include <dftracer/utils/core/runtime.h>
2
#include <dftracer/utils/core/tasks/coro_scope.h>
3
#include <dftracer/utils/core/utilities/utility_executor.h>
4
#include <dftracer/utils/python/py_dict_helpers.h>
5
#include <dftracer/utils/python/py_runtime_mixin.h>
6
#include <dftracer/utils/python/py_type_helpers.h>
7
#include <dftracer/utils/python/runtime.h>
8
#include <dftracer/utils/python/utilities/reorganization_planner.h>
9
#include <dftracer/utils/utilities/composites/dft/reorganize/reorganization_planner.h>
10

11
#include <string>
12
#include <vector>
13

14
using dftracer::utils::CoroScope;
15
using dftracer::utils::Runtime;
16
using dftracer::utils::utilities::behaviors::UtilityExecutor;
17
namespace tags = dftracer::utils::utilities::tags;
18
using namespace dftracer::utils::utilities::composites::dft::reorganize;
19

20
static Runtime *get_runtime(ReorganizationPlannerObject *self) {
8✔
21
    return resolve_runtime(self);
8✔
22
}
23

24
static void ReorganizationPlanner_dealloc(ReorganizationPlannerObject *self) {
8✔
25
    runtime_backed_dealloc(self);
8✔
26
}
8✔
27

28
static PyObject *ReorganizationPlanner_new(PyTypeObject *type, PyObject *args,
8✔
29
                                           PyObject *kwds) {
30
    return runtime_backed_new<ReorganizationPlannerObject>(type, args, kwds);
8✔
31
}
32

33
static int ReorganizationPlanner_init(ReorganizationPlannerObject *self,
8✔
34
                                      PyObject *args, PyObject *kwds) {
35
    return runtime_backed_init(self, args, kwds);
8✔
36
}
37

38
static PyObject *ReorganizationPlanner_plan(ReorganizationPlannerObject *self,
8✔
39
                                            PyObject *args, PyObject *kwds) {
40
    using dftracer::utils::coro::CoroTask;
41

42
    static const char *kwlist[] = {"source_files", "groups", "index_dir", NULL};
43
    PyObject *source_files_obj;
44
    PyObject *groups_obj = Py_None;
8✔
45
    const char *index_dir = "";
8✔
46

47
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Os", (char **)kwlist,
8!
48
                                     &source_files_obj, &groups_obj,
49
                                     &index_dir))
50
        return NULL;
×
51

52
    if (!PyList_Check(source_files_obj)) {
8✔
53
        PyErr_SetString(PyExc_TypeError, "source_files must be a list");
×
54
        return NULL;
×
55
    }
56

57
    Py_ssize_t nfiles = PyList_Size(source_files_obj);
8!
58
    std::vector<std::string> files;
8✔
59
    files.reserve(static_cast<std::size_t>(nfiles));
8!
60
    for (Py_ssize_t i = 0; i < nfiles; i++) {
16✔
61
        PyObject *item = PyList_GetItem(source_files_obj, i);
8!
62
        const char *s = PyUnicode_AsUTF8(item);
8!
63
        if (!s) return NULL;
8✔
64
        files.emplace_back(s);
8!
65
    }
4✔
66

67
    std::vector<PredicateGroup> groups;
8✔
68
    if (groups_obj && groups_obj != Py_None) {
8!
69
        if (!PyList_Check(groups_obj)) {
8!
70
            PyErr_SetString(PyExc_TypeError,
×
71
                            "groups must be a list of dicts or None");
72
            return NULL;
×
73
        }
74
        Py_ssize_t n = PyList_Size(groups_obj);
8!
75
        groups.reserve(static_cast<std::size_t>(n));
8!
76
        for (Py_ssize_t i = 0; i < n; i++) {
16✔
77
            PyObject *item = PyList_GetItem(groups_obj, i);
8!
78
            PredicateGroup g;
8✔
79
            PyObject *name = PyDict_GetItemString(item, "name");
8!
80
            PyObject *pred = PyDict_GetItemString(item, "query");
8!
81
            if (name) {
8!
82
                const char *ns = PyUnicode_AsUTF8(name);
8!
83
                if (!ns) return NULL;
8!
84
                g.name = ns;
8!
85
            }
4✔
86
            if (pred) {
8!
87
                const char *ps = PyUnicode_AsUTF8(pred);
8!
88
                if (!ps) return NULL;
8✔
89
                g.query = ps;
8!
90
            }
4✔
91
            groups.push_back(std::move(g));
8!
92
        }
8✔
93
    }
4✔
94

95
    ReorganizationPlannerInput input;
8✔
96
    input.source_files = std::move(files);
8✔
97
    input.groups = std::move(groups);
8✔
98
    input.index_dir = index_dir;
8!
99

100
    ExtractionPlan plan;
8✔
101
    auto *plan_p = &plan;
8✔
102
    ReorganizationPlannerInput input_copy = input;
8!
103

104
    if (!run_blocking([&] {
12!
105
            Runtime *rt = get_runtime(self);
8!
106
            auto task = run_coro_scope(
4!
107
                rt->executor(),
4✔
108
                [plan_p, input_copy](CoroScope &scope) -> CoroTask<void> {
36!
109
                    auto planner =
12✔
110
                        std::make_shared<ReorganizationPlannerUtility>();
12!
111
                    UtilityExecutor<ReorganizationPlannerInput, ExtractionPlan,
12✔
112
                                    tags::NeedsContext>
113
                        exec(planner);
12!
114
                    *plan_p = co_await exec.execute(scope, input_copy);
20!
115
                });
24!
116
            rt->submit(std::move(task), "reorganization-planner").wait();
8!
117
        })) {
8✔
UNCOV
118
        return NULL;
×
119
    }
120

121
    // groups list
122
    PyObject *py_groups =
4✔
123
        PyList_New(static_cast<Py_ssize_t>(plan.groups.size()));
8!
124
    if (!py_groups) return NULL;
8✔
125
    for (std::size_t i = 0; i < plan.groups.size(); i++) {
24✔
126
        PyObject *g = PyDict_New();
16!
127
        if (!g) {
16!
128
            Py_DECREF(py_groups);
×
129
            return NULL;
×
130
        }
131
        dict_set_steal(g, "name",
16!
132
                       PyUnicode_FromString(plan.groups[i].name.c_str()));
16!
133
        dict_set_steal(g, "query",
16!
134
                       PyUnicode_FromString(plan.groups[i].query.c_str()));
16!
135
        PyList_SetItem(py_groups, static_cast<Py_ssize_t>(i), g);
16!
136
    }
8✔
137

138
    // source_files list
139
    PyObject *py_sources =
4✔
140
        PyList_New(static_cast<Py_ssize_t>(plan.source_files.size()));
8!
141
    if (!py_sources) {
8!
142
        Py_DECREF(py_groups);
×
143
        return NULL;
×
144
    }
145
    for (std::size_t i = 0; i < plan.source_files.size(); i++) {
16✔
146
        const auto &sf = plan.source_files[i];
8✔
147
        PyObject *entry = PyDict_New();
8!
148
        if (!entry) {
8!
149
            Py_DECREF(py_groups);
×
150
            Py_DECREF(py_sources);
×
151
            return NULL;
×
152
        }
153
        dict_set_steal(entry, "file_path",
8!
154
                       PyUnicode_FromString(sf.file_path.c_str()));
4!
155
        dict_set_steal(entry, "index_path",
8!
156
                       PyUnicode_FromString(sf.index_path.c_str()));
4!
157
        dict_set_steal(entry, "num_checkpoints",
8!
158
                       PyLong_FromSize_t(sf.num_checkpoints));
8!
159
        dict_set_steal(entry, "uncompressed_size",
8!
160
                       PyLong_FromUnsignedLongLong(sf.uncompressed_size));
8!
161
        dict_set_steal(entry, "checkpoint_size",
8!
162
                       PyLong_FromUnsignedLongLong(sf.checkpoint_size));
8!
163
        PyList_SetItem(py_sources, static_cast<Py_ssize_t>(i), entry);
8!
164
    }
4✔
165

166
    // tasks list
167
    PyObject *py_tasks = PyList_New(static_cast<Py_ssize_t>(plan.tasks.size()));
8!
168
    if (!py_tasks) {
8!
169
        Py_DECREF(py_groups);
×
170
        Py_DECREF(py_sources);
×
171
        return NULL;
×
172
    }
173
    for (std::size_t i = 0; i < plan.tasks.size(); i++) {
20✔
174
        const auto &t = plan.tasks[i];
12✔
175
        PyObject *entry = PyDict_New();
12!
176
        if (!entry) {
12!
177
            Py_DECREF(py_groups);
×
178
            Py_DECREF(py_sources);
×
179
            Py_DECREF(py_tasks);
×
180
            return NULL;
×
181
        }
182
        dict_set_steal(entry, "source_file_idx",
12!
183
                       PyLong_FromSize_t(t.source_file_idx));
12!
184
        dict_set_steal(entry, "checkpoint_idx",
12!
185
                       PyLong_FromUnsignedLongLong(t.checkpoint_idx));
12!
186
        dict_set_steal(entry, "target_group",
12!
187
                       PyUnicode_FromString(t.target_group.c_str()));
6!
188
        dict_set_steal(entry, "start_byte",
12!
189
                       PyLong_FromUnsignedLongLong(t.start_byte));
12!
190
        dict_set_steal(entry, "end_byte",
12!
191
                       PyLong_FromUnsignedLongLong(t.end_byte));
12!
192
        PyList_SetItem(py_tasks, static_cast<Py_ssize_t>(i), entry);
12!
193
    }
6✔
194

195
    PyObject *result = PyDict_New();
8!
196
    if (!result) {
8!
197
        Py_DECREF(py_groups);
×
198
        Py_DECREF(py_sources);
×
199
        Py_DECREF(py_tasks);
×
200
        return NULL;
×
201
    }
202
    PyDict_SetItemString(result, "groups", py_groups);
8!
203
    Py_DECREF(py_groups);
4!
204
    PyDict_SetItemString(result, "source_files", py_sources);
8!
205
    Py_DECREF(py_sources);
4!
206
    PyDict_SetItemString(result, "tasks", py_tasks);
8!
207
    Py_DECREF(py_tasks);
4!
208
    dict_set_steal(result, "total_events",
8!
209
                   PyLong_FromSize_t(plan.total_events));
4!
210
    return result;
8✔
211
}
8✔
212

213
static PyObject *ReorganizationPlanner_call(PyObject *self, PyObject *args,
2✔
214
                                            PyObject *kwds) {
215
    return ReorganizationPlanner_plan((ReorganizationPlannerObject *)self, args,
3✔
216
                                      kwds);
2✔
217
}
218

219
static PyMethodDef ReorganizationPlanner_methods[] = {
220
    {"process", (PyCFunction)ReorganizationPlanner_plan,
221
     METH_VARARGS | METH_KEYWORDS,
222
     "process(source_files, groups=None, index_dir='')\n"
223
     "--\n"
224
     "\n"
225
     "Build a reorganization plan for trace files.\n"
226
     "\n"
227
     "Args:\n"
228
     "    source_files (list[str]): Paths to source trace files.\n"
229
     "    groups (list[dict] or None): Predicate group definitions\n"
230
     "        (default None).\n"
231
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
232
     "\n"
233
     "Returns:\n"
234
     "    dict: Extraction plan.\n"},
235
    {NULL} /* Sentinel */
236
};
237

238
PyTypeObject ReorganizationPlannerType = {
239
    PyVarObject_HEAD_INIT(
240
        NULL,
241
        0) "dftracer_utils_ext.ReorganizationPlannerUtility", /* tp_name */
242
    sizeof(ReorganizationPlannerObject),                      /* tp_basicsize */
243
    0,                                                        /* tp_itemsize */
244
    (destructor)ReorganizationPlanner_dealloc,                /* tp_dealloc */
245
    0,                                        /* tp_vectorcall_offset */
246
    0,                                        /* tp_getattr */
247
    0,                                        /* tp_setattr */
248
    0,                                        /* tp_as_async */
249
    0,                                        /* tp_repr */
250
    0,                                        /* tp_as_number */
251
    0,                                        /* tp_as_sequence */
252
    0,                                        /* tp_as_mapping */
253
    0,                                        /* tp_hash */
254
    ReorganizationPlanner_call,               /* tp_call */
255
    0,                                        /* tp_str */
256
    0,                                        /* tp_getattro */
257
    0,                                        /* tp_setattro */
258
    0,                                        /* tp_as_buffer */
259
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
260
    "ReorganizationPlannerUtility(runtime: Runtime | None = None)\n"
261
    "--\n"
262
    "\n"
263
    "Plan semantic reorganization of trace files into predicate groups.\n"
264
    "\n"
265
    "Args:\n"
266
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
267
    "        If None, uses the default global Runtime.\n", /* tp_doc */
268
    0,                                                     /* tp_traverse */
269
    0,                                                     /* tp_clear */
270
    0,                                                     /* tp_richcompare */
271
    0,                                    /* tp_weaklistoffset */
272
    0,                                    /* tp_iter */
273
    0,                                    /* tp_iternext */
274
    ReorganizationPlanner_methods,        /* tp_methods */
275
    0,                                    /* tp_members */
276
    0,                                    /* tp_getset */
277
    0,                                    /* tp_base */
278
    0,                                    /* tp_dict */
279
    0,                                    /* tp_descr_get */
280
    0,                                    /* tp_descr_set */
281
    0,                                    /* tp_dictoffset */
282
    (initproc)ReorganizationPlanner_init, /* tp_init */
283
    0,                                    /* tp_alloc */
284
    ReorganizationPlanner_new,            /* tp_new */
285
};
286

287
int init_reorganization_planner(PyObject *m) {
2✔
288
    if (register_type(m, &ReorganizationPlannerType,
2✔
289
                      "ReorganizationPlannerUtility") < 0)
2✔
UNCOV
290
        return -1;
×
291

292
    return 0;
2✔
293
}
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc