26043728131

Committed 18 May 2026 03:37PM UTC coverage: 51.706% (-0.4%) from 52.076%

Build # 26043728131

Build Type

push

github

Committed by

hariharan-devarajan

Commit Message

feat(perf): performance improvements for parallel reading, indexing, and aggregation

Indexer
- Streaming parse-and-emit worker pipeline with bounded memory usage
- Concurrent SST artifact ingestion with staging support
- Gzip member slicing for parallel indexing
- Lazy decoding for compressed value counts
- Bypass DOM wrapper for indexer hot path (simdjson on_demand)
- Decoupled write workers from parse workers
- --rebuild-summaries flag and optimized root summary rebuild

Aggregator / MPI
- Task-based DAG execution for aggregator pipeline
- Shared staging for multi-node artifact relocation
- Per-node thread scaling to avoid oversubscription
- Unified distributed aggregation tracking, removed manifest consolidation
- Deterministic aggregation and intra-file parallelism

Trace reader / query
- Compiled predicate evaluation for AND-of-EQ queries
- Uniform-match shortcut for AND-of-EQ queries
- Line-range support for work items and checkpoint processing
- Optimized chunk pruning and checkpoint handling

Replay
- Pipelined replay with coroutines and channels
- JsonParser-based trace processing
- Optimized string handling and i/o buffering

Organize / writer / dft
- Parallel slice creation and merging in organize visitor
- Inline indexer in organize
- Gzip member tracking in writer
- Coroutine-based event dispatcher with extracted parse logic
- Batch flushing in organize visitor

Arrow / call_tree
- Optimized arrow conversion
- Arrow IPC support and improved save/load in call_tree

Build / infrastructure
- zlib-ng option, system simdjson fallback
- cgroup v1/v2 memory limit detection
- Auto-computed per-file memory estimates and batch sizes
- CI: perf branch trigger, formatting

Docs
- Rewritten indexer and trace reader API references

Coverage Stats

35907 of 90345 branches covered (39.74%)

Branch coverage included in aggregate %.

16869 of 21880 new or added lines in 137 files covered. (77.1%)

273 existing lines in 39 files now uncovered.

32021 of 41028 relevant lines covered (78.05%)

13164.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.57

/src/dftracer/utils/python/memoryview_batch.cpp

1	#define PY_SSIZE_T_CLEAN
2	#include <dftracer/utils/python/memoryview_batch.h>
3
4	#include <cstring>
5
6	namespace dftracer::utils::python {
7
NEW 8	static void MemoryViewBatch_dealloc(MemoryViewBatchObject *self) {	×
NEW 9	delete self->data;	×
NEW 10	Py_TYPE(self)->tp_free((PyObject *)self);	×
NEW 11	}	×
12
NEW 13	static int MemoryViewBatch_getbuffer(MemoryViewBatchObject *self,	×
14	Py_buffer *view, int flags) {
NEW 15	if (!self->data \|\| self->data->buffer.empty()) {	×
NEW 16	PyErr_SetString(PyExc_BufferError, "MemoryViewBatch has no data");	×
NEW 17	return -1;	×
18	}
NEW 19	return PyBuffer_FillInfo(view, (PyObject *)self, self->data->buffer.data(),	×
NEW 20	static_cast<Py_ssize_t>(self->data->buffer.size()),	×
NEW 21	1, flags);	×
22	}
23
NEW 24	static Py_ssize_t MemoryViewBatch_length(MemoryViewBatchObject *self) {	×
NEW 25	if (!self->data) return 0;	×
NEW 26	return static_cast<Py_ssize_t>(self->data->num_entries());	×
27	}
28
29	PyObject MemoryViewBatch_item(MemoryViewBatchObject self, Py_ssize_t i) {	2,302✔
30	if (!self->data) {	2,302✔
NEW 31	PyErr_SetString(PyExc_IndexError, "MemoryViewBatch has no data");	×
NEW 32	return NULL;	×
33	}
34	Py_ssize_t n = static_cast<Py_ssize_t>(self->data->num_entries());	2,302✔
35	if (i < 0 \|\| i >= n) {	2,302!
NEW 36	PyErr_SetString(PyExc_IndexError, "MemoryViewBatch index out of range");	×
NEW 37	return NULL;	×
38	}
39
40	Py_buffer buf;
41	std::memset(&buf, 0, sizeof(buf));	2,302✔
42	buf.buf = self->data->buffer.data() + self->data->offsets[i];	2,302✔
43	buf.obj = (PyObject *)self;	2,302!
44	Py_INCREF(self);	1,151✔
45	buf.len = self->data->lengths[i];	2,302✔
46	buf.itemsize = 1;	2,302✔
47	buf.readonly = 1;	2,302✔
48	buf.ndim = 1;	2,302✔
49	buf.format = const_cast<char *>("B");	2,302✔
50	buf.shape = &buf.len;	2,302✔
51	buf.strides = &buf.itemsize;	2,302✔
52	buf.suboffsets = NULL;	2,302✔
53	buf.internal = NULL;	2,302✔
54	return PyMemoryView_FromBuffer(&buf);	2,302!
55	}	1,151✔
56
57	static PyBufferProcs MemoryViewBatch_as_buffer = {
58	(getbufferproc)MemoryViewBatch_getbuffer,
59	NULL,
60	};
61
62	static PySequenceMethods MemoryViewBatch_as_sequence = {
63	(lenfunc)MemoryViewBatch_length,
64	NULL,
65	NULL,
66	(ssizeargfunc)MemoryViewBatch_item,
67	};
68
NEW 69	static PyObject MemoryViewBatch_get_num_entries(MemoryViewBatchObject self,	×
70	void *) {
NEW 71	if (!self->data) return PyLong_FromLong(0);	×
NEW 72	return PyLong_FromSsize_t(	×
NEW 73	static_cast<Py_ssize_t>(self->data->num_entries()));	×
74	}
75
NEW 76	static PyObject MemoryViewBatch_get_num_bytes(MemoryViewBatchObject self,	×
77	void *) {
NEW 78	if (!self->data) return PyLong_FromLong(0);	×
NEW 79	return PyLong_FromSsize_t(	×
NEW 80	static_cast<Py_ssize_t>(self->data->buffer.size()));	×
81	}
82
83	static PyGetSetDef MemoryViewBatch_getsetters[] = {
84	{"num_entries", (getter)MemoryViewBatch_get_num_entries, NULL,
85	"Number of entries", NULL},
86	{"num_bytes", (getter)MemoryViewBatch_get_num_bytes, NULL,
87	"Total buffer size in bytes", NULL},
88	{NULL}};
89
90	PyTypeObject MemoryViewBatchType = {
91	.ob_base = PyVarObject_HEAD_INIT(NULL, 0).tp_name =
92	"dftracer_utils_ext._MemoryViewBatch",
93	.tp_basicsize = sizeof(MemoryViewBatchObject),
94	.tp_itemsize = 0,
95	.tp_dealloc = (destructor)MemoryViewBatch_dealloc,
96	.tp_as_sequence = &MemoryViewBatch_as_sequence,
97	.tp_as_buffer = &MemoryViewBatch_as_buffer,
98	.tp_flags = Py_TPFLAGS_DEFAULT,
99	.tp_doc = "Zero-copy batch of byte entries backed by a contiguous buffer",
100	.tp_getset = MemoryViewBatch_getsetters,
101	};
102
103	int init_memoryview_batch(PyObject *m) {	2✔
104	if (PyType_Ready(&MemoryViewBatchType) < 0) return -1;	2✔
105	Py_INCREF(&MemoryViewBatchType);	1✔
106	if (PyModule_AddObject(m, "_MemoryViewBatch",	3!
107	(PyObject *)&MemoryViewBatchType) < 0) {	2!
108	Py_DECREF(&MemoryViewBatchType);
NEW 109	return -1;	×
110	}
111	return 0;	2✔
112	}	1✔
113
114	} // namespace dftracer::utils::python

llnl / dftracer-utils / 26043728131

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous