• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

llnl / dftracer-utils / 29787659139

20 Jul 2026 11:34PM UTC coverage: 51.578% (-1.1%) from 52.66%
29787659139

Pull #99

github

web-flow
Merge ee805adeb into 06bc84ec9
Pull Request #99: Support CM time_metric (NS/MS/SEC/US) across reader and viz

34832 of 86278 branches covered (40.37%)

Branch coverage included in aggregate %.

1034 of 1319 new or added lines in 30 files covered. (78.39%)

5193 existing lines in 197 files now uncovered.

35349 of 49791 relevant lines covered (70.99%)

9765.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.01
/src/dftracer/utils/core/rocksdb/database.cpp
1
#include <dftracer/utils/core/common/constants.h>
2
#include <dftracer/utils/core/common/error.h>
3
#include <dftracer/utils/core/common/filesystem.h>
4
#include <dftracer/utils/core/env.h>
5
#include <dftracer/utils/core/rocksdb/database.h>
6
#include <dftracer/utils/core/rocksdb/filesystem.h>
7
#include <rocksdb/slice.h>
8
#include <rocksdb/table.h>
9

10
#include <algorithm>
11
#include <atomic>
12
#include <cstdlib>
13
#include <stdexcept>
14
#include <utility>
15

16
namespace dftracer::utils::rocksdb {
17

18
namespace {
19

20
std::atomic<bool>& process_exiting_flag() {
6,264✔
21
    static std::atomic<bool> flag{false};
22
    return flag;
6,264✔
23
}
24

25
const ::rocksdb::ReadOptions& read_options() {
12,800✔
26
    static const ::rocksdb::ReadOptions options;
12,800!
27
    return options;
12,800✔
UNCOV
28
}
×
29

30
const ::rocksdb::WriteOptions& write_options() {
2,217✔
31
    static const auto options = [] {
2,324!
32
        ::rocksdb::WriteOptions wo;
107✔
33
        wo.disableWAL = true;
107✔
34
        return wo;
107✔
35
    }();
36
    return options;
2,217✔
UNCOV
37
}
×
38

39
void cleanup_failed_open(::rocksdb::DB*& db,
×
40
                         std::vector<::rocksdb::ColumnFamilyHandle*>& handles) {
41
    if (db != nullptr) {
×
42
        for (auto* handle : handles) {
×
43
            if (handle != nullptr) {
×
44
                db->DestroyColumnFamilyHandle(handle);
×
UNCOV
45
            }
×
46
        }
47
        static_cast<void>(db->Close());
×
48
        delete db;
×
49
        db = nullptr;
×
UNCOV
50
    }
×
51
    handles.clear();
×
52
}
×
53

54
}  // namespace
55

56
void mark_process_exiting_for_rocksdb() {
5✔
57
    process_exiting_flag().store(true, std::memory_order_relaxed);
5✔
58
}
5✔
59

60
RocksDatabase::RocksDatabase() = default;
25,112✔
61

62
RocksDatabase::RocksDatabase(const std::string& db_path, OpenMode open_mode) {
12✔
63
    open(db_path, open_mode);
3!
64
}
6✔
65

66
RocksDatabase::~RocksDatabase() { close(); }
12,562!
67

68
RocksDatabase::RocksDatabase(RocksDatabase&& other) noexcept
×
69
    : db_path_(std::move(other.db_path_)),
×
70
      open_mode_(other.open_mode_),
×
71
      file_system_(std::move(other.file_system_)),
×
72
      env_(std::move(other.env_)),
×
73
      db_(std::exchange(other.db_, nullptr)),
×
74
      column_families_(std::move(other.column_families_)) {}
×
75

76
RocksDatabase& RocksDatabase::operator=(RocksDatabase&& other) noexcept {
×
77
    if (this != &other) {
×
78
        close();
×
79
        db_path_ = std::move(other.db_path_);
×
80
        open_mode_ = other.open_mode_;
×
81
        file_system_ = std::move(other.file_system_);
×
82
        env_ = std::move(other.env_);
×
83
        db_ = std::exchange(other.db_, nullptr);
×
84
        column_families_ = std::move(other.column_families_);
×
UNCOV
85
    }
×
86
    return *this;
×
87
}
88

89
const decltype(cf::ALL)& RocksDatabase::default_column_families() {
1,530✔
90
    return cf::ALL;
1,530✔
91
}
92

93
::rocksdb::Options RocksDatabase::default_options() {
6,517✔
94
    ::rocksdb::Options options;
6,517✔
95
    options.create_if_missing = true;
6,517✔
96
    options.create_missing_column_families = true;
6,517✔
97
    options.allow_concurrent_memtable_write = true;
6,517✔
98
    options.enable_pipelined_write = true;
6,517✔
99
    options.max_open_files = Env::rocksdb_max_open_files();
6,517!
100
    options.max_background_jobs = 8;
6,517✔
101
    options.max_subcompactions = 8;
6,517✔
102
    options.write_buffer_size = 256 * 1024 * 1024;
6,517✔
103
    options.max_write_buffer_number = 4;
6,517✔
104
    return options;
6,517✔
105
}
6,517!
106

107
::rocksdb::ColumnFamilyOptions RocksDatabase::default_column_family_options() {
6,517✔
108
    ::rocksdb::ColumnFamilyOptions options;
6,517✔
109

110
    ::rocksdb::BlockBasedTableOptions bbt;
6,517✔
111
    bbt.block_size = 32 * 1024;
6,517✔
112
    bbt.format_version = 5;
6,517✔
113
    bbt.index_block_restart_interval = 16;
6,517✔
114
    options.table_factory.reset(::rocksdb::NewBlockBasedTableFactory(bbt));
6,517!
115

116
#ifdef DFTRACER_UTILS_ENABLE_ZSTD
117
    options.compression = ::rocksdb::kZSTD;
118
    options.compression_opts.level = constants::rocksdb::ZSTD_COMPRESSION_LEVEL;
119
    options.compression_opts.max_dict_bytes =
120
        constants::rocksdb::ZSTD_MAX_DICT_BYTES;
121
    options.compression_opts.zstd_max_train_bytes =
122
        constants::rocksdb::ZSTD_MAX_TRAIN_BYTES;
123
    options.compression_opts.enabled = true;
124
    options.bottommost_compression = ::rocksdb::kZSTD;
125
    options.bottommost_compression_opts.level =
126
        constants::rocksdb::ZSTD_COMPRESSION_LEVEL;
127
    options.bottommost_compression_opts.max_dict_bytes =
128
        constants::rocksdb::ZSTD_MAX_DICT_BYTES;
129
    options.bottommost_compression_opts.zstd_max_train_bytes =
130
        constants::rocksdb::ZSTD_MAX_TRAIN_BYTES;
131
    options.bottommost_compression_opts.enabled = true;
132
#elif defined(DFTRACER_UTILS_ENABLE_LZ4)
133
    options.compression = ::rocksdb::kLZ4Compression;
134
    options.bottommost_compression = ::rocksdb::kZlibCompression;
135
#else
136
    options.compression = ::rocksdb::kZlibCompression;
6,517✔
137
    options.bottommost_compression = ::rocksdb::kZlibCompression;
6,517✔
138
#endif
139
    return options;
6,517✔
140
}
6,517!
141

142
bool RocksDatabase::open(const std::string& db_path, OpenMode open_mode) {
6,285✔
143
    close();
6,285✔
144
    db_path_ = db_path;
6,285✔
145
    open_mode_ = open_mode;
6,285✔
146

147
    std::error_code ec;
6,285✔
148
    if (open_mode_ == OpenMode::ReadWrite) {
6,285✔
149
        fs::create_directories(fs::path(db_path_), ec);
967!
150
    }
945✔
151

152
    auto db_options = default_options();
6,285✔
153
    if (open_mode_ == OpenMode::ReadOnly) {
6,285✔
154
        db_options.create_if_missing = false;
5,336✔
155
        db_options.create_missing_column_families = false;
5,336✔
156
    }
5,336✔
157
    file_system_ = make_dftracer_file_system();
6,285✔
158
    env_ = make_dftracer_env(file_system_);
6,281!
159
    db_options.env = env_.get();
6,281✔
160
    auto cf_options = default_column_family_options();
6,281✔
161

162
    std::vector<std::string> column_family_names;
6,285✔
163
    auto list_status = ::rocksdb::DB::ListColumnFamilies(db_options, db_path_,
6,285!
164
                                                         &column_family_names);
165
    if (!list_status.ok()) {
6,285!
166
        if (open_mode_ == OpenMode::ReadOnly) {
607✔
167
            throw DFTUtilsException(
44!
168
                ErrorCode::IO, "Failed to list RocksDB column families at '" +
22!
169
                                   db_path_ + "': " + list_status.ToString());
22!
170
        }
171
        column_family_names.reserve(default_column_families().size());
585!
172
        for (auto name : default_column_families()) {
15,795✔
173
            column_family_names.emplace_back(name);
15,210!
174
        }
175
    } else {
585✔
176
        if (open_mode_ == OpenMode::ReadWrite) {
5,678✔
177
            for (const auto& name : default_column_families()) {
9,720✔
178
                if (std::find(column_family_names.begin(),
18,720!
179
                              column_family_names.end(),
9,360✔
180
                              name) == column_family_names.end()) {
9,360✔
181
                    column_family_names.emplace_back(name);
×
UNCOV
182
                }
×
183
            }
184
        }
360✔
185
    }
186

187
    std::vector<::rocksdb::ColumnFamilyDescriptor> descriptors;
6,263✔
188
    descriptors.reserve(column_family_names.size());
6,263✔
189
    for (const auto& name : column_family_names) {
168,993✔
190
        auto opts = cf_options;
162,734✔
191
        if (cf_options_override_) {
162,738✔
192
            cf_options_override_(name, opts);
161,850✔
193
        }
161,848✔
194
        descriptors.emplace_back(name, opts);
162,736✔
195
    }
162,738✔
196

197
    std::vector<::rocksdb::ColumnFamilyHandle*> handles;
6,259✔
198
    ::rocksdb::Status status;
6,259!
199
    if (open_mode_ == OpenMode::ReadOnly) {
6,259✔
200
        status = ::rocksdb::DB::OpenForReadOnly(
5,314!
201
            db_options, db_path_, descriptors, &handles, &db_, false);
5,314✔
202
    } else {
5,314✔
203
        status = ::rocksdb::DB::Open(db_options, db_path_, descriptors,
1,890!
204
                                     &handles, &db_);
945✔
205
    }
206
    if (!status.ok()) {
6,259!
207
        cleanup_failed_open(db_, handles);
×
208
        throw DFTUtilsException(ErrorCode::IO, "Failed to open RocksDB at '" +
×
209
                                                   db_path_ +
×
210
                                                   "': " + status.ToString());
×
211
    }
212

213
    column_families_.clear();
6,259✔
214
    for (std::size_t i = 0; i < descriptors.size(); ++i) {
168,993✔
215
        column_families_.emplace(descriptors[i].name, handles[i]);
162,734!
216
    }
162,734✔
217

218
    return true;
219
}
6,323✔
220

221
void RocksDatabase::close() {
12,562✔
222
    if (db_ == nullptr) {
12,562✔
223
        column_families_.clear();
6,303✔
224
        return;
6,303✔
225
    }
226

227
    if (process_exiting_flag().load(std::memory_order_relaxed)) {
6,259!
228
        db_ = nullptr;
×
229
        column_families_.clear();
×
230
        env_.reset();
×
231
        file_system_.reset();
×
232
        db_path_.clear();
×
233
        return;
×
234
    }
235

236
    for (auto& entry : column_families_) {
168,993✔
237
        if (entry.second != nullptr) {
162,734!
238
            db_->DestroyColumnFamilyHandle(entry.second);
162,734✔
239
            entry.second = nullptr;
162,734✔
240
        }
162,734✔
241
    }
242
    column_families_.clear();
6,259✔
243

244
    auto* db = db_;
6,259✔
245
    db_ = nullptr;
6,259✔
246
    static_cast<void>(db->Close());
6,259✔
247
    delete db;
6,259!
248
    env_.reset();
6,259✔
249
    file_system_.reset();
6,259✔
250
    db_path_.clear();
6,259✔
251
}
12,562✔
252

253
bool RocksDatabase::is_open() const noexcept { return db_ != nullptr; }
18✔
254

255
bool RocksDatabase::is_read_only() const noexcept {
1,025✔
256
    return open_mode_ == OpenMode::ReadOnly;
1,025✔
257
}
258

259
const std::string& RocksDatabase::path() const noexcept { return db_path_; }
×
260

261
::rocksdb::DB* RocksDatabase::get() const noexcept { return db_; }
×
262

263
::rocksdb::ColumnFamilyHandle* RocksDatabase::column_family_handle(
44,425✔
264
    std::string_view column_family) const {
265
    const auto name = column_family.empty() ? std::string(cf::DEFAULT)
44,425!
266
                                            : std::string(column_family);
44,425✔
267
    const auto it = column_families_.find(name);
44,425✔
268
    if (it == column_families_.end() || it->second == nullptr) {
44,427!
269
        throw DFTUtilsException(ErrorCode::INVALID_ARGUMENT,
×
270
                                "Unknown RocksDB column family: " + name);
×
271
    }
272
    return it->second;
44,412✔
273
}
44,431✔
274

275
::rocksdb::Status RocksDatabase::put(std::string_view key,
758✔
276
                                     std::string_view value,
277
                                     std::string_view column_family) {
278
    return db_->Put(write_options(), column_family_handle(column_family),
1,516✔
279
                    ::rocksdb::Slice(key.data(), key.size()),
758✔
280
                    ::rocksdb::Slice(value.data(), value.size()));
758✔
281
}
282

283
::rocksdb::Status RocksDatabase::get(std::string_view key, std::string* value,
7,428✔
284
                                     std::string_view column_family) const {
285
    return db_->Get(read_options(), column_family_handle(column_family),
14,856✔
286
                    ::rocksdb::Slice(key.data(), key.size()), value);
7,428✔
287
}
288

289
::rocksdb::Status RocksDatabase::merge(std::string_view key,
×
290
                                       std::string_view value,
291
                                       std::string_view column_family) {
292
    return db_->Merge(write_options(), column_family_handle(column_family),
×
293
                      ::rocksdb::Slice(key.data(), key.size()),
×
294
                      ::rocksdb::Slice(value.data(), value.size()));
×
295
}
296

297
void RocksDatabase::set_cf_options_override(CfOptionsOverride override) {
6,247✔
298
    cf_options_override_ = std::move(override);
6,247✔
299
}
6,247✔
300

301
::rocksdb::Status RocksDatabase::merge(Batch& batch,
970✔
302
                                       std::string_view column_family,
303
                                       std::string_view key,
304
                                       std::string_view value) {
305
    return batch.Merge(column_family_handle(column_family),
1,940✔
306
                       ::rocksdb::Slice(key.data(), key.size()),
970✔
307
                       ::rocksdb::Slice(value.data(), value.size()));
970✔
308
}
309

310
::rocksdb::Status RocksDatabase::del(std::string_view key,
×
311
                                     std::string_view column_family) {
312
    return db_->Delete(write_options(), column_family_handle(column_family),
×
313
                       ::rocksdb::Slice(key.data(), key.size()));
×
314
}
315

316
::rocksdb::Status RocksDatabase::delete_range(std::string_view begin_key,
×
317
                                              std::string_view end_key,
318
                                              std::string_view column_family) {
319
    return db_->DeleteRange(
×
UNCOV
320
        write_options(), column_family_handle(column_family),
×
321
        ::rocksdb::Slice(begin_key.data(), begin_key.size()),
×
322
        ::rocksdb::Slice(end_key.data(), end_key.size()));
×
323
}
324

325
::rocksdb::Status RocksDatabase::put(Batch& batch,
29,593✔
326
                                     std::string_view column_family,
327
                                     std::string_view key,
328
                                     std::string_view value) {
329
    return batch.Put(column_family_handle(column_family),
59,186✔
330
                     ::rocksdb::Slice(key.data(), key.size()),
29,593✔
331
                     ::rocksdb::Slice(value.data(), value.size()));
29,593✔
332
}
333

334
::rocksdb::Status RocksDatabase::del(Batch& batch,
60✔
335
                                     std::string_view column_family,
336
                                     std::string_view key) {
337
    return batch.Delete(column_family_handle(column_family),
120✔
338
                        ::rocksdb::Slice(key.data(), key.size()));
60✔
339
}
340

341
RocksDatabase::Batch RocksDatabase::begin_batch() const { return Batch(); }
1,459✔
342

343
::rocksdb::Status RocksDatabase::commit_batch(Batch& batch) {
1,459✔
344
    return db_->Write(write_options(), &batch);
1,459✔
345
}
346

347
std::unique_ptr<::rocksdb::Iterator> RocksDatabase::new_iterator(
5,373✔
348
    std::string_view column_family) const {
349
    return std::unique_ptr<::rocksdb::Iterator>(
5,373✔
350
        db_->NewIterator(read_options(), column_family_handle(column_family)));
5,373✔
351
}
352

353
::rocksdb::Status RocksDatabase::compact(std::string_view column_family) {
24✔
354
    ::rocksdb::CompactRangeOptions opts;
24✔
355
    opts.max_subcompactions = 8;
24✔
356
    return db_->CompactRange(opts, column_family_handle(column_family), nullptr,
24✔
357
                             nullptr);
358
}
359

360
::rocksdb::Status RocksDatabase::ingest_external_files(
235✔
361
    std::string_view column_family,
362
    const std::vector<std::string>& external_files, bool ingest_behind) {
363
    if (external_files.empty()) {
235✔
364
        return ::rocksdb::Status::OK();
13✔
365
    }
366
    ::rocksdb::IngestExternalFileOptions opts;
222✔
367
    opts.move_files = false;
222✔
368
    opts.snapshot_consistency = true;
222✔
369
    opts.allow_global_seqno = true;
222✔
370
    opts.allow_blocking_flush = true;
222✔
371
    opts.ingest_behind = ingest_behind;
222✔
372
    return db_->IngestExternalFile(column_family_handle(column_family),
444✔
373
                                   external_files, opts);
222✔
374
}
235✔
375

376
}  // namespace dftracer::utils::rocksdb
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc