26043728131

Committed 18 May 2026 03:37PM UTC coverage: 51.706% (-0.4%) from 52.076%

Build # 26043728131

Build Type

push

github

Committed by

hariharan-devarajan

Commit Message

feat(perf): performance improvements for parallel reading, indexing, and aggregation

Indexer
- Streaming parse-and-emit worker pipeline with bounded memory usage
- Concurrent SST artifact ingestion with staging support
- Gzip member slicing for parallel indexing
- Lazy decoding for compressed value counts
- Bypass DOM wrapper for indexer hot path (simdjson on_demand)
- Decoupled write workers from parse workers
- --rebuild-summaries flag and optimized root summary rebuild

Aggregator / MPI
- Task-based DAG execution for aggregator pipeline
- Shared staging for multi-node artifact relocation
- Per-node thread scaling to avoid oversubscription
- Unified distributed aggregation tracking, removed manifest consolidation
- Deterministic aggregation and intra-file parallelism

Trace reader / query
- Compiled predicate evaluation for AND-of-EQ queries
- Uniform-match shortcut for AND-of-EQ queries
- Line-range support for work items and checkpoint processing
- Optimized chunk pruning and checkpoint handling

Replay
- Pipelined replay with coroutines and channels
- JsonParser-based trace processing
- Optimized string handling and i/o buffering

Organize / writer / dft
- Parallel slice creation and merging in organize visitor
- Inline indexer in organize
- Gzip member tracking in writer
- Coroutine-based event dispatcher with extracted parse logic
- Batch flushing in organize visitor

Arrow / call_tree
- Optimized arrow conversion
- Arrow IPC support and improved save/load in call_tree

Build / infrastructure
- zlib-ng option, system simdjson fallback
- cgroup v1/v2 memory limit detection
- Auto-computed per-file memory estimates and batch sizes
- CI: perf branch trigger, formatting

Docs
- Rewritten indexer and trace reader API references

Coverage Stats

35907 of 90345 branches covered (39.74%)

Branch coverage included in aggregate %.

16869 of 21880 new or added lines in 137 files covered. (77.1%)

273 existing lines in 39 files now uncovered.

32021 of 41028 relevant lines covered (78.05%)

13164.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.69

/src/dftracer/utils/core/utils/timer.cpp

#include <dftracer/utils/core/common/logging.h>
#include <dftracer/utils/core/utils/timer.h>

#include <algorithm>
#include <cinttypes>
#include <cstdio>
#include <utility>
#include <vector>

namespace dftracer::utils {

Timer::Timer(bool autostart, bool verbose)
    : verbose_(verbose), running_(false) {
    if (autostart) {
        start();
    }
}

Timer::Timer(const std::string& name, bool autostart, bool verbose)
    : verbose_(verbose), running_(false), name_(name) {
    if (autostart) {
        start();
    }
}

Timer::~Timer() {
    stop();
    if (verbose_) {
        if (name_.empty()) {
            std::printf("Elapsed time: %" PRId64 " ns\n", elapsed());
        } else {
            std::printf("[%s] Elapsed time: %" PRId64 " ns\n", name_.c_str(),
                        elapsed());
        }
    }
}

void Timer::start() {
    start_time = Clock::now();
    running_ = true;
}

void Timer::stop() {
    if (running_) {
        end_time = Clock::now();
        running_ = false;
    }
}

std::int64_t Timer::elapsed() const {
    if (running_) {
        return std::chrono::duration_cast<std::chrono::nanoseconds>(
                   Clock::now() - start_time)
            .count();
    } else {
        return std::chrono::duration_cast<std::chrono::nanoseconds>(end_time -
                                                                    start_time)
            .count();
    }
}

void Timer::increment(const std::string& key, std::uint64_t by) {
    counters_[key] += by;
}

void Timer::set_counter(const std::string& key, std::uint64_t value) {
    counters_[key] = value;
}

const std::unordered_map<std::string, std::uint64_t>& Timer::counters() const {
    return counters_;
}

void Timer::print_stages(const std::string& prefix) const {
    if (counters_.empty()) return;

    std::vector<std::pair<std::string, std::uint64_t>> sorted(counters_.begin(),
                                                              counters_.end());
    std::sort(sorted.begin(), sorted.end());

    std::uint64_t total_ns = 0;
    for (const auto& [_, ns] : sorted) total_ns += ns;

    if (!name_.empty()) {
        std::printf("%s%s (%.2f ms)\n", prefix.c_str(), name_.c_str(),
                    static_cast<double>(total_ns) / 1e6);
    }
    for (std::size_t i = 0; i < sorted.size(); ++i) {
        const auto& [key, ns] = sorted[i];
        bool last = (i + 1 == sorted.size());
        double ms = static_cast<double>(ns) / 1e6;
        double pct = total_ns > 0 ? 100.0 * ns / total_ns : 0.0;
        std::printf("%s%s %-28s %8.2f ms  (%5.1f%%)\n", prefix.c_str(),
                    last ? "\\-- " : "|-- ", key.c_str(), ms, pct);
    }
}

}  // namespace dftracer::utils

1	#include <dftracer/utils/core/common/logging.h>
2	#include <dftracer/utils/core/utils/timer.h>
3
4	#include <algorithm>
5	#include <cinttypes>
6	#include <cstdio>
7	#include <utility>
8	#include <vector>
9
10	namespace dftracer::utils {
11
12	Timer::Timer(bool autostart, bool verbose)	128!
13	: verbose_(verbose), running_(false) {	96✔
14	if (autostart) {	64✔
15	start();	64!
16	}	32✔
17	}	96✔
18
19	Timer::Timer(const std::string& name, bool autostart, bool verbose)	12!
20	: verbose_(verbose), running_(false), name_(name) {	9✔
21	if (autostart) {	6✔
22	start();	6!
23	}	3✔
24	}	9✔
25
26	Timer::~Timer() {	123✔
27	stop();	82!
28	if (verbose_) {	82✔
29	if (name_.empty()) {	×
30	std::printf("Elapsed time: %" PRId64 " ns\n", elapsed());	×
31	} else {
32	std::printf("[%s] Elapsed time: %" PRId64 " ns\n", name_.c_str(),	×
33	elapsed());	×
34	}
35	}
36	}	123✔
37
38	void Timer::start() {	70✔
39	start_time = Clock::now();	70✔
40	running_ = true;	70✔
41	}	70✔
42
43	void Timer::stop() {	104✔
44	if (running_) {	104✔
45	end_time = Clock::now();	82✔
46	running_ = false;	82✔
47	}	41✔
48	}	104✔
49
50	std::int64_t Timer::elapsed() const {	36✔
51	if (running_) {	36✔
52	return std::chrono::duration_cast<std::chrono::nanoseconds>(	21!
53	Clock::now() - start_time)	21!
54	.count();	14✔
55	} else {
56	return std::chrono::duration_cast<std::chrono::nanoseconds>(end_time -	55!
57	start_time)	22✔
58	.count();	22✔
59	}
60	}	18✔
61
62	void Timer::increment(const std::string& key, std::uint64_t by) {	16✔
63	counters_[key] += by;	16✔
64	}	16✔
65
NEW 66	void Timer::set_counter(const std::string& key, std::uint64_t value) {	×
NEW 67	counters_[key] = value;	×
NEW 68	}	×
69
70	const std::unordered_map<std::string, std::uint64_t>& Timer::counters() const {	6✔
71	return counters_;	6✔
72	}
73
NEW 74	void Timer::print_stages(const std::string& prefix) const {	×
NEW 75	if (counters_.empty()) return;	×
76
77	std::vector<std::pair<std::string, std::uint64_t>> sorted(counters_.begin(),
NEW 78	counters_.end());	×
NEW 79	std::sort(sorted.begin(), sorted.end());	×
80
NEW 81	std::uint64_t total_ns = 0;	×
NEW 82	for (const auto& [_, ns] : sorted) total_ns += ns;	×
83
NEW 84	if (!name_.empty()) {	×
NEW 85	std::printf("%s%s (%.2f ms)\n", prefix.c_str(), name_.c_str(),	×
NEW 86	static_cast<double>(total_ns) / 1e6);	×
87	}
NEW 88	for (std::size_t i = 0; i < sorted.size(); ++i) {	×
NEW 89	const auto& [key, ns] = sorted[i];	×
NEW 90	bool last = (i + 1 == sorted.size());	×
NEW 91	double ms = static_cast<double>(ns) / 1e6;	×
NEW 92	double pct = total_ns > 0 ? 100.0 * ns / total_ns : 0.0;	×
NEW 93	std::printf("%s%s %-28s %8.2f ms (%5.1f%%)\n", prefix.c_str(),	×
94	last ? "\\-- " : "\|-- ", key.c_str(), ms, pct);	×
95	}
NEW 96	}	×
97
98	} // namespace dftracer::utils

llnl / dftracer-utils / 26043728131

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous