• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

llnl / dftracer-utils / 28496595030

01 Jul 2026 05:50AM UTC coverage: 50.727% (-1.6%) from 52.278%
28496595030

Pull #83

github

web-flow
Merge 8f1ff4df5 into 2efed6649
Pull Request #83: refactor and improve code QoL

31872 of 80367 branches covered (39.66%)

Branch coverage included in aggregate %.

770 of 1591 new or added lines in 85 files covered. (48.4%)

5070 existing lines in 182 files now uncovered.

32742 of 47009 relevant lines covered (69.65%)

9887.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.82
/src/dftracer/utils/python/utilities/reorganization_planner.cpp
1
#include <dftracer/utils/core/runtime.h>
2
#include <dftracer/utils/core/tasks/coro_scope.h>
3
#include <dftracer/utils/core/utilities/utility_executor.h>
4
#include <dftracer/utils/python/py_dict_helpers.h>
5
#include <dftracer/utils/python/py_runtime_mixin.h>
6
#include <dftracer/utils/python/py_type_helpers.h>
7
#include <dftracer/utils/python/runtime.h>
8
#include <dftracer/utils/python/utilities/reorganization_planner.h>
9
#include <dftracer/utils/utilities/composites/dft/reorganize/reorganization_planner.h>
10

11
#include <string>
12
#include <vector>
13

14
using dftracer::utils::CoroScope;
15
using dftracer::utils::Runtime;
16
using dftracer::utils::utilities::behaviors::UtilityExecutor;
17
namespace tags = dftracer::utils::utilities::tags;
18
using namespace dftracer::utils::utilities::composites::dft::reorganize;
19

20
static Runtime *get_runtime(ReorganizationPlannerObject *self) {
4✔
21
    return resolve_runtime(self);
4✔
22
}
23

24
static void ReorganizationPlanner_dealloc(ReorganizationPlannerObject *self) {
4✔
25
    runtime_backed_dealloc(self);
4✔
26
}
4✔
27

28
static PyObject *ReorganizationPlanner_new(PyTypeObject *type, PyObject *args,
4✔
29
                                           PyObject *kwds) {
30
    return runtime_backed_new<ReorganizationPlannerObject>(type, args, kwds);
4✔
31
}
32

33
static int ReorganizationPlanner_init(ReorganizationPlannerObject *self,
4✔
34
                                      PyObject *args, PyObject *kwds) {
35
    return runtime_backed_init(self, args, kwds);
4✔
36
}
37

38
static PyObject *ReorganizationPlanner_plan(ReorganizationPlannerObject *self,
4✔
39
                                            PyObject *args, PyObject *kwds) {
40
    using dftracer::utils::coro::CoroTask;
41

42
    static const char *kwlist[] = {"source_files", "groups", "index_dir", NULL};
43
    PyObject *source_files_obj;
44
    PyObject *groups_obj = Py_None;
4✔
45
    const char *index_dir = "";
4✔
46

47
    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Os", (char **)kwlist,
4!
48
                                     &source_files_obj, &groups_obj,
49
                                     &index_dir))
50
        return NULL;
×
51

52
    if (!PyList_Check(source_files_obj)) {
4!
53
        PyErr_SetString(PyExc_TypeError, "source_files must be a list");
×
54
        return NULL;
×
55
    }
56

57
    Py_ssize_t nfiles = PyList_Size(source_files_obj);
4✔
58
    std::vector<std::string> files;
4✔
59
    files.reserve(static_cast<std::size_t>(nfiles));
4!
60
    for (Py_ssize_t i = 0; i < nfiles; i++) {
8✔
61
        PyObject *item = PyList_GetItem(source_files_obj, i);
4!
62
        const char *s = PyUnicode_AsUTF8(item);
4!
63
        if (!s) return NULL;
4!
64
        files.emplace_back(s);
4!
65
    }
4✔
66

67
    std::vector<PredicateGroup> groups;
4✔
68
    if (groups_obj && groups_obj != Py_None) {
4!
69
        if (!PyList_Check(groups_obj)) {
4!
70
            PyErr_SetString(PyExc_TypeError,
×
71
                            "groups must be a list of dicts or None");
72
            return NULL;
×
73
        }
74
        Py_ssize_t n = PyList_Size(groups_obj);
4!
75
        groups.reserve(static_cast<std::size_t>(n));
4!
76
        for (Py_ssize_t i = 0; i < n; i++) {
8✔
77
            PyObject *item = PyList_GetItem(groups_obj, i);
4!
78
            PredicateGroup g;
4✔
79
            PyObject *name = PyDict_GetItemString(item, "name");
4!
80
            PyObject *pred = PyDict_GetItemString(item, "query");
4!
81
            if (name) {
4!
82
                const char *ns = PyUnicode_AsUTF8(name);
4!
83
                if (!ns) return NULL;
4!
84
                g.name = ns;
4!
85
            }
4✔
86
            if (pred) {
4!
87
                const char *ps = PyUnicode_AsUTF8(pred);
4!
88
                if (!ps) return NULL;
4!
89
                g.query = ps;
4!
90
            }
4✔
91
            groups.push_back(std::move(g));
4!
92
        }
4!
93
    }
4✔
94

95
    ReorganizationPlannerInput input;
4✔
96
    input.source_files = std::move(files);
4✔
97
    input.groups = std::move(groups);
4✔
98
    input.index_dir = index_dir;
4!
99

100
    ExtractionPlan plan;
4✔
101
    auto *plan_p = &plan;
4✔
102
    ReorganizationPlannerInput input_copy = input;
4!
103

104
    if (!run_blocking([&] {
8!
105
            Runtime *rt = get_runtime(self);
4✔
106
            auto task = run_coro_scope(
4!
107
                rt->executor(),
4✔
108
                [plan_p, input_copy](CoroScope &scope) -> CoroTask<void> {
32!
109
                    auto planner =
12✔
110
                        std::make_shared<ReorganizationPlannerUtility>();
12!
111
                    UtilityExecutor<ReorganizationPlannerInput, ExtractionPlan,
12✔
112
                                    tags::NeedsContext>
113
                        exec(planner);
12!
114
                    *plan_p = co_await exec.execute(scope, input_copy);
20!
115
                });
12!
116
            rt->submit(std::move(task), "reorganization-planner").wait();
4!
117
        })) {
4✔
UNCOV
118
        return NULL;
×
119
    }
120

121
    // groups list
122
    PyObject *py_groups =
4✔
123
        PyList_New(static_cast<Py_ssize_t>(plan.groups.size()));
4!
124
    if (!py_groups) return NULL;
4!
125
    for (std::size_t i = 0; i < plan.groups.size(); i++) {
12✔
126
        PyObject *g = PyDict_New();
8!
127
        if (!g) {
8!
UNCOV
128
            Py_DECREF(py_groups);
×
129
            return NULL;
×
130
        }
131
        dict_set_steal(g, "name",
8!
132
                       PyUnicode_FromString(plan.groups[i].name.c_str()));
8!
133
        dict_set_steal(g, "query",
8!
134
                       PyUnicode_FromString(plan.groups[i].query.c_str()));
8!
135
        PyList_SetItem(py_groups, static_cast<Py_ssize_t>(i), g);
8!
136
    }
8✔
137

138
    // source_files list
139
    PyObject *py_sources =
4✔
140
        PyList_New(static_cast<Py_ssize_t>(plan.source_files.size()));
4!
141
    if (!py_sources) {
4!
UNCOV
142
        Py_DECREF(py_groups);
×
143
        return NULL;
×
144
    }
145
    for (std::size_t i = 0; i < plan.source_files.size(); i++) {
8✔
146
        const auto &sf = plan.source_files[i];
4✔
147
        PyObject *entry = PyDict_New();
4!
148
        if (!entry) {
4!
UNCOV
149
            Py_DECREF(py_groups);
×
UNCOV
150
            Py_DECREF(py_sources);
×
151
            return NULL;
×
152
        }
153
        dict_set_steal(entry, "file_path",
4!
154
                       PyUnicode_FromString(sf.file_path.c_str()));
4!
155
        dict_set_steal(entry, "index_path",
4!
156
                       PyUnicode_FromString(sf.index_path.c_str()));
4!
157
        dict_set_steal(entry, "num_checkpoints",
4!
158
                       PyLong_FromSize_t(sf.num_checkpoints));
4!
159
        dict_set_steal(entry, "uncompressed_size",
4!
160
                       PyLong_FromUnsignedLongLong(sf.uncompressed_size));
4!
161
        dict_set_steal(entry, "checkpoint_size",
4!
162
                       PyLong_FromUnsignedLongLong(sf.checkpoint_size));
4!
163
        PyList_SetItem(py_sources, static_cast<Py_ssize_t>(i), entry);
4!
164
    }
4✔
165

166
    // tasks list
167
    PyObject *py_tasks = PyList_New(static_cast<Py_ssize_t>(plan.tasks.size()));
4!
168
    if (!py_tasks) {
4!
UNCOV
169
        Py_DECREF(py_groups);
×
UNCOV
170
        Py_DECREF(py_sources);
×
171
        return NULL;
×
172
    }
173
    for (std::size_t i = 0; i < plan.tasks.size(); i++) {
10✔
174
        const auto &t = plan.tasks[i];
6✔
175
        PyObject *entry = PyDict_New();
6!
176
        if (!entry) {
6!
UNCOV
177
            Py_DECREF(py_groups);
×
UNCOV
178
            Py_DECREF(py_sources);
×
UNCOV
179
            Py_DECREF(py_tasks);
×
180
            return NULL;
×
181
        }
182
        dict_set_steal(entry, "source_file_idx",
6!
183
                       PyLong_FromSize_t(t.source_file_idx));
6!
184
        dict_set_steal(entry, "checkpoint_idx",
6!
185
                       PyLong_FromUnsignedLongLong(t.checkpoint_idx));
6!
186
        dict_set_steal(entry, "target_group",
6!
187
                       PyUnicode_FromString(t.target_group.c_str()));
6!
188
        dict_set_steal(entry, "start_byte",
6!
189
                       PyLong_FromUnsignedLongLong(t.start_byte));
6!
190
        dict_set_steal(entry, "end_byte",
6!
191
                       PyLong_FromUnsignedLongLong(t.end_byte));
6!
192
        PyList_SetItem(py_tasks, static_cast<Py_ssize_t>(i), entry);
6!
193
    }
6✔
194

195
    PyObject *result = PyDict_New();
4!
196
    if (!result) {
4!
UNCOV
197
        Py_DECREF(py_groups);
×
UNCOV
198
        Py_DECREF(py_sources);
×
UNCOV
199
        Py_DECREF(py_tasks);
×
200
        return NULL;
×
201
    }
202
    PyDict_SetItemString(result, "groups", py_groups);
4!
203
    Py_DECREF(py_groups);
4!
204
    PyDict_SetItemString(result, "source_files", py_sources);
4!
205
    Py_DECREF(py_sources);
4!
206
    PyDict_SetItemString(result, "tasks", py_tasks);
4!
207
    Py_DECREF(py_tasks);
4!
208
    dict_set_steal(result, "total_events",
4!
209
                   PyLong_FromSize_t(plan.total_events));
4!
210
    return result;
4✔
211
}
4✔
212

213
static PyObject *ReorganizationPlanner_call(PyObject *self, PyObject *args,
1✔
214
                                            PyObject *kwds) {
215
    return ReorganizationPlanner_plan((ReorganizationPlannerObject *)self, args,
2✔
216
                                      kwds);
1✔
217
}
218

219
static PyMethodDef ReorganizationPlanner_methods[] = {
220
    {"process", (PyCFunction)ReorganizationPlanner_plan,
221
     METH_VARARGS | METH_KEYWORDS,
222
     "process(source_files, groups=None, index_dir='')\n"
223
     "--\n"
224
     "\n"
225
     "Build a reorganization plan for trace files.\n"
226
     "\n"
227
     "Args:\n"
228
     "    source_files (list[str]): Paths to source trace files.\n"
229
     "    groups (list[dict] or None): Predicate group definitions\n"
230
     "        (default None).\n"
231
     "    index_dir (str): Directory for .dftindex stores (default '').\n"
232
     "\n"
233
     "Returns:\n"
234
     "    dict: Extraction plan.\n"},
235
    {NULL} /* Sentinel */
236
};
237

238
PyTypeObject ReorganizationPlannerType = {
239
    PyVarObject_HEAD_INIT(
240
        NULL,
241
        0) "dftracer_utils_ext.ReorganizationPlannerUtility", /* tp_name */
242
    sizeof(ReorganizationPlannerObject),                      /* tp_basicsize */
243
    0,                                                        /* tp_itemsize */
244
    (destructor)ReorganizationPlanner_dealloc,                /* tp_dealloc */
245
    0,                                        /* tp_vectorcall_offset */
246
    0,                                        /* tp_getattr */
247
    0,                                        /* tp_setattr */
248
    0,                                        /* tp_as_async */
249
    0,                                        /* tp_repr */
250
    0,                                        /* tp_as_number */
251
    0,                                        /* tp_as_sequence */
252
    0,                                        /* tp_as_mapping */
253
    0,                                        /* tp_hash */
254
    ReorganizationPlanner_call,               /* tp_call */
255
    0,                                        /* tp_str */
256
    0,                                        /* tp_getattro */
257
    0,                                        /* tp_setattro */
258
    0,                                        /* tp_as_buffer */
259
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
260
    "ReorganizationPlannerUtility(runtime: Runtime | None = None)\n"
261
    "--\n"
262
    "\n"
263
    "Plan semantic reorganization of trace files into predicate groups.\n"
264
    "\n"
265
    "Args:\n"
266
    "    runtime (Runtime or None): Runtime for thread pool control.\n"
267
    "        If None, uses the default global Runtime.\n", /* tp_doc */
268
    0,                                                     /* tp_traverse */
269
    0,                                                     /* tp_clear */
270
    0,                                                     /* tp_richcompare */
271
    0,                                    /* tp_weaklistoffset */
272
    0,                                    /* tp_iter */
273
    0,                                    /* tp_iternext */
274
    ReorganizationPlanner_methods,        /* tp_methods */
275
    0,                                    /* tp_members */
276
    0,                                    /* tp_getset */
277
    0,                                    /* tp_base */
278
    0,                                    /* tp_dict */
279
    0,                                    /* tp_descr_get */
280
    0,                                    /* tp_descr_set */
281
    0,                                    /* tp_dictoffset */
282
    (initproc)ReorganizationPlanner_init, /* tp_init */
283
    0,                                    /* tp_alloc */
284
    ReorganizationPlanner_new,            /* tp_new */
285
};
286

287
int init_reorganization_planner(PyObject *m) {
1✔
288
    if (register_type(m, &ReorganizationPlannerType,
2!
289
                      "ReorganizationPlannerUtility") < 0)
1✔
UNCOV
290
        return -1;
×
291

292
    return 0;
1✔
293
}
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc