26043728131

Committed 18 May 2026 03:37PM UTC coverage: 51.706% (-0.4%) from 52.076%

Build # 26043728131

Build Type

push

github

Committed by

hariharan-devarajan

Commit Message

feat(perf): performance improvements for parallel reading, indexing, and aggregation

Indexer
- Streaming parse-and-emit worker pipeline with bounded memory usage
- Concurrent SST artifact ingestion with staging support
- Gzip member slicing for parallel indexing
- Lazy decoding for compressed value counts
- Bypass DOM wrapper for indexer hot path (simdjson on_demand)
- Decoupled write workers from parse workers
- --rebuild-summaries flag and optimized root summary rebuild

Aggregator / MPI
- Task-based DAG execution for aggregator pipeline
- Shared staging for multi-node artifact relocation
- Per-node thread scaling to avoid oversubscription
- Unified distributed aggregation tracking, removed manifest consolidation
- Deterministic aggregation and intra-file parallelism

Trace reader / query
- Compiled predicate evaluation for AND-of-EQ queries
- Uniform-match shortcut for AND-of-EQ queries
- Line-range support for work items and checkpoint processing
- Optimized chunk pruning and checkpoint handling

Replay
- Pipelined replay with coroutines and channels
- JsonParser-based trace processing
- Optimized string handling and i/o buffering

Organize / writer / dft
- Parallel slice creation and merging in organize visitor
- Inline indexer in organize
- Gzip member tracking in writer
- Coroutine-based event dispatcher with extracted parse logic
- Batch flushing in organize visitor

Arrow / call_tree
- Optimized arrow conversion
- Arrow IPC support and improved save/load in call_tree

Build / infrastructure
- zlib-ng option, system simdjson fallback
- cgroup v1/v2 memory limit detection
- Auto-computed per-file memory estimates and batch sizes
- CI: perf branch trigger, formatting

Docs
- Rewritten indexer and trace reader API references

Coverage Stats

35907 of 90345 branches covered (39.74%)

Branch coverage included in aggregate %.

16869 of 21880 new or added lines in 137 files covered. (77.1%)

273 existing lines in 39 files now uncovered.

32021 of 41028 relevant lines covered (78.05%)

13164.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.59

/src/dftracer/utils/python/dftracer_utils_ext.cpp

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <dftracer/utils/core/common/config.h>
#include <dftracer/utils/python/batch_indexer.h>
#include <dftracer/utils/python/index_database.h>
#include <dftracer/utils/python/indexer.h>
#include <dftracer/utils/python/indexer_checkpoint.h>
#include <dftracer/utils/python/json.h>
#include <dftracer/utils/python/memoryview_batch.h>
#include <dftracer/utils/python/runtime.h>
#include <dftracer/utils/python/sst_distribution.h>
#include <dftracer/utils/python/task_handle.h>
#include <dftracer/utils/python/trace_reader.h>
#include <dftracer/utils/python/trace_reader_iterator.h>
#include <dftracer/utils/python/utilities/aggregator.h>
#include <dftracer/utils/python/utilities/comparator.h>
#include <dftracer/utils/python/utilities/metadata_collector.h>
#include <dftracer/utils/python/utilities/reconstruction_planner.h>
#include <dftracer/utils/python/utilities/reorganization_planner.h>
#include <dftracer/utils/python/utilities/statistics_aggregator.h>
#include <dftracer/utils/python/utilities/statistics_query.h>
#ifdef DFTRACER_UTILS_ENABLE_ARROW
#include <dftracer/utils/python/arrow_stream_capsule.h>
#include <dftracer/utils/python/streaming_iterator.h>
#endif
#ifdef DFTRACER_UTILS_ENABLE_ARROW_IPC
#include <dftracer/utils/python/arrow_parallel_reader.h>
#endif

static PyModuleDef dftracer_utils_module = {
    PyModuleDef_HEAD_INIT,
    "dftracer_utils_ext",   /* m_name */
    "DFTracer utils module with indexer, reader, "
    "and utility bindings", /* m_doc */
    -1,                     /* m_size */
    NULL,                   /* m_methods */
    NULL,                   /* m_slots */
    NULL,                   /* m_traverse */
    NULL,                   /* m_clear */
    NULL                    /* m_free */
};

PyMODINIT_FUNC PyInit_dftracer_utils_ext(void) {
    PyObject *m;
    m = PyModule_Create(&dftracer_utils_module);
    if (m == NULL) return NULL;
    if (init_indexer_checkpoint(m) < 0) return NULL;
    if (init_checkpoint_indexer(m) < 0) return NULL;
    if (init_indexer(m) < 0) return NULL;
    if (init_task_handle(m) < 0) return NULL;
    if (init_runtime(m) < 0) return NULL;
    if (dftracer::utils::python::init_memoryview_batch(m) < 0) return NULL;
    if (init_json_dict_value(m) < 0) return NULL;
    if (init_trace_reader_iterator(m) < 0) return NULL;
#ifdef DFTRACER_UTILS_ENABLE_ARROW
    if (dftracer::utils::python::init_arrow_streaming_iterator(m) < 0)
        return NULL;
    if (init_arrow_batch_stream(m) < 0) return NULL;
#endif
#ifdef DFTRACER_UTILS_ENABLE_ARROW_IPC
    if (dftracer::utils::python::init_arrow_parallel_reader(m) < 0) return NULL;
#endif
    if (init_trace_reader(m) < 0) return NULL;
    if (init_statistics_query(m) < 0) return NULL;
    if (init_statistics_aggregator(m) < 0) return NULL;
    if (init_metadata_collector(m) < 0) return NULL;
    if (init_reorganization_planner(m) < 0) return NULL;
    if (init_reconstruction_planner(m) < 0) return NULL;
    if (init_aggregator(m) < 0) return NULL;
    if (init_comparator(m) < 0) return NULL;
    if (init_index_database(m) < 0) return NULL;
    if (init_sst_distribution(m) < 0) return NULL;
    return m;
}

1	#define PY_SSIZE_T_CLEAN
2	#include <Python.h>
3	#include <dftracer/utils/core/common/config.h>
4	#include <dftracer/utils/python/batch_indexer.h>
5	#include <dftracer/utils/python/index_database.h>
6	#include <dftracer/utils/python/indexer.h>
7	#include <dftracer/utils/python/indexer_checkpoint.h>
8	#include <dftracer/utils/python/json.h>
9	#include <dftracer/utils/python/memoryview_batch.h>
10	#include <dftracer/utils/python/runtime.h>
11	#include <dftracer/utils/python/sst_distribution.h>
12	#include <dftracer/utils/python/task_handle.h>
13	#include <dftracer/utils/python/trace_reader.h>
14	#include <dftracer/utils/python/trace_reader_iterator.h>
15	#include <dftracer/utils/python/utilities/aggregator.h>
16	#include <dftracer/utils/python/utilities/comparator.h>
17	#include <dftracer/utils/python/utilities/metadata_collector.h>
18	#include <dftracer/utils/python/utilities/reconstruction_planner.h>
19	#include <dftracer/utils/python/utilities/reorganization_planner.h>
20	#include <dftracer/utils/python/utilities/statistics_aggregator.h>
21	#include <dftracer/utils/python/utilities/statistics_query.h>
22	#ifdef DFTRACER_UTILS_ENABLE_ARROW
23	#include <dftracer/utils/python/arrow_stream_capsule.h>
24	#include <dftracer/utils/python/streaming_iterator.h>
25	#endif
26	#ifdef DFTRACER_UTILS_ENABLE_ARROW_IPC
27	#include <dftracer/utils/python/arrow_parallel_reader.h>
28	#endif
29
30	static PyModuleDef dftracer_utils_module = {
31	PyModuleDef_HEAD_INIT,
32	"dftracer_utils_ext", /* m_name */
33	"DFTracer utils module with indexer, reader, "
34	"and utility bindings", /* m_doc */
35	-1, /* m_size */
36	NULL, /* m_methods */
37	NULL, /* m_slots */
38	NULL, /* m_traverse */
39	NULL, /* m_clear */
40	NULL /* m_free */
41	};
42
43	PyMODINIT_FUNC PyInit_dftracer_utils_ext(void) {	2✔
44	PyObject *m;
45	m = PyModule_Create(&dftracer_utils_module);	2✔
46	if (m == NULL) return NULL;	2✔
47	if (init_indexer_checkpoint(m) < 0) return NULL;	2✔
48	if (init_checkpoint_indexer(m) < 0) return NULL;	2✔
49	if (init_indexer(m) < 0) return NULL;	2✔
50	if (init_task_handle(m) < 0) return NULL;	2✔
51	if (init_runtime(m) < 0) return NULL;	2✔
52	if (dftracer::utils::python::init_memoryview_batch(m) < 0) return NULL;	2✔
53	if (init_json_dict_value(m) < 0) return NULL;	2✔
54	if (init_trace_reader_iterator(m) < 0) return NULL;	2✔
55	#ifdef DFTRACER_UTILS_ENABLE_ARROW
56	if (dftracer::utils::python::init_arrow_streaming_iterator(m) < 0)	2✔
NEW 57	return NULL;	×
58	if (init_arrow_batch_stream(m) < 0) return NULL;	2✔
59	#endif
60	#ifdef DFTRACER_UTILS_ENABLE_ARROW_IPC
61	if (dftracer::utils::python::init_arrow_parallel_reader(m) < 0) return NULL;	2✔
62	#endif
63	if (init_trace_reader(m) < 0) return NULL;	2✔
64	if (init_statistics_query(m) < 0) return NULL;	2✔
65	if (init_statistics_aggregator(m) < 0) return NULL;	2✔
66	if (init_metadata_collector(m) < 0) return NULL;	2✔
67	if (init_reorganization_planner(m) < 0) return NULL;	2✔
68	if (init_reconstruction_planner(m) < 0) return NULL;	2✔
69	if (init_aggregator(m) < 0) return NULL;	2✔
70	if (init_comparator(m) < 0) return NULL;	2✔
71	if (init_index_database(m) < 0) return NULL;	2✔
72	if (init_sst_distribution(m) < 0) return NULL;	2✔
73	return m;	2✔
74	}	1✔

llnl / dftracer-utils / 26043728131

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous