• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

dgraph-io / badger / 4348167445

06 Mar 2023 09:49PM UTC coverage: 60.652% (-0.5%) from 61.177%
4348167445

push

GitHub
docs: add loggie to projects using badger (#1882)

8535 of 14072 relevant lines covered (60.65%)

465147.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.65
/level_handler.go
1
/*
2
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
3
 *
4
 * Licensed under the Apache License, Version 2.0 (the "License");
5
 * you may not use this file except in compliance with the License.
6
 * You may obtain a copy of the License at
7
 *
8
 *     http://www.apache.org/licenses/LICENSE-2.0
9
 *
10
 * Unless required by applicable law or agreed to in writing, software
11
 * distributed under the License is distributed on an "AS IS" BASIS,
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
 * See the License for the specific language governing permissions and
14
 * limitations under the License.
15
 */
16

17
package badger
18

19
import (
20
        "fmt"
21
        "sort"
22
        "sync"
23

24
        "github.com/dgraph-io/badger/v4/table"
25
        "github.com/dgraph-io/badger/v4/y"
26
)
27

28
type levelHandler struct {
29
        // Guards tables, totalSize.
30
        sync.RWMutex
31

32
        // For level >= 1, tables are sorted by key ranges, which do not overlap.
33
        // For level 0, tables are sorted by time.
34
        // For level 0, newest table are at the back. Compact the oldest one first, which is at the front.
35
        tables         []*table.Table
36
        totalSize      int64
37
        totalStaleSize int64
38

39
        // The following are initialized once and const.
40
        level    int
41
        strLevel string
42
        db       *DB
43
}
44

45
func (s *levelHandler) isLastLevel() bool {
198✔
46
        return s.level == s.db.opt.MaxLevels-1
198✔
47
}
198✔
48

49
func (s *levelHandler) getTotalStaleSize() int64 {
2✔
50
        s.RLock()
2✔
51
        defer s.RUnlock()
2✔
52
        return s.totalStaleSize
2✔
53
}
2✔
54

55
func (s *levelHandler) getTotalSize() int64 {
839,246✔
56
        s.RLock()
839,246✔
57
        defer s.RUnlock()
839,246✔
58
        return s.totalSize
839,246✔
59
}
839,246✔
60

61
// initTables replaces s.tables with given tables. This is done during loading.
62
func (s *levelHandler) initTables(tables []*table.Table) {
2,030✔
63
        s.Lock()
2,030✔
64
        defer s.Unlock()
2,030✔
65

2,030✔
66
        s.tables = tables
2,030✔
67
        s.totalSize = 0
2,030✔
68
        s.totalStaleSize = 0
2,030✔
69
        for _, t := range tables {
2,102✔
70
                s.addSize(t)
72✔
71
        }
72✔
72

73
        if s.level == 0 {
2,320✔
74
                // Key range will overlap. Just sort by fileID in ascending order
290✔
75
                // because newer tables are at the end of level 0.
290✔
76
                sort.Slice(s.tables, func(i, j int) bool {
296✔
77
                        return s.tables[i].ID() < s.tables[j].ID()
6✔
78
                })
6✔
79
        } else {
1,740✔
80
                // Sort tables by keys.
1,740✔
81
                sort.Slice(s.tables, func(i, j int) bool {
1,830✔
82
                        return y.CompareKeys(s.tables[i].Smallest(), s.tables[j].Smallest()) < 0
90✔
83
                })
90✔
84
        }
85
}
86

87
// deleteTables remove tables idx0, ..., idx1-1.
88
func (s *levelHandler) deleteTables(toDel []*table.Table) error {
161✔
89
        s.Lock() // s.Unlock() below
161✔
90

161✔
91
        toDelMap := make(map[uint64]struct{})
161✔
92
        for _, t := range toDel {
333✔
93
                toDelMap[t.ID()] = struct{}{}
172✔
94
        }
172✔
95

96
        // Make a copy as iterators might be keeping a slice of tables.
97
        var newTables []*table.Table
161✔
98
        for _, t := range s.tables {
1,347✔
99
                _, found := toDelMap[t.ID()]
1,186✔
100
                if !found {
2,200✔
101
                        newTables = append(newTables, t)
1,014✔
102
                        continue
1,014✔
103
                }
104
                s.subtractSize(t)
172✔
105
        }
106
        s.tables = newTables
161✔
107

161✔
108
        s.Unlock() // Unlock s _before_ we DecrRef our tables, which can be slow.
161✔
109

161✔
110
        return decrRefs(toDel)
161✔
111
}
112

113
// replaceTables will replace tables[left:right] with newTables. Note this EXCLUDES tables[right].
114
// You must call decr() to delete the old tables _after_ writing the update to the manifest.
115
func (s *levelHandler) replaceTables(toDel, toAdd []*table.Table) error {
174✔
116
        // Need to re-search the range of tables in this level to be replaced as other goroutines might
174✔
117
        // be changing it as well.  (They can't touch our tables, but if they add/remove other tables,
174✔
118
        // the indices get shifted around.)
174✔
119
        s.Lock() // We s.Unlock() below.
174✔
120

174✔
121
        toDelMap := make(map[uint64]struct{})
174✔
122
        for _, t := range toDel {
404✔
123
                toDelMap[t.ID()] = struct{}{}
230✔
124
        }
230✔
125
        var newTables []*table.Table
174✔
126
        for _, t := range s.tables {
5,158✔
127
                _, found := toDelMap[t.ID()]
4,984✔
128
                if !found {
9,738✔
129
                        newTables = append(newTables, t)
4,754✔
130
                        continue
4,754✔
131
                }
132
                s.subtractSize(t)
230✔
133
        }
134

135
        // Increase totalSize first.
136
        for _, t := range toAdd {
603✔
137
                s.addSize(t)
429✔
138
                t.IncrRef()
429✔
139
                newTables = append(newTables, t)
429✔
140
        }
429✔
141

142
        // Assign tables.
143
        s.tables = newTables
174✔
144
        sort.Slice(s.tables, func(i, j int) bool {
8,372✔
145
                return y.CompareKeys(s.tables[i].Smallest(), s.tables[j].Smallest()) < 0
8,198✔
146
        })
8,198✔
147
        s.Unlock() // s.Unlock before we DecrRef tables -- that can be slow.
174✔
148
        return decrRefs(toDel)
174✔
149
}
150

151
// addTable adds toAdd table to levelHandler. Normally when we add tables to levelHandler, we sort
152
// tables based on table.Smallest. This is required for correctness of the system. But in case of
153
// stream writer this can be avoided. We can just add tables to levelHandler's table list
154
// and after all addTable calls, we can sort table list(check sortTable method).
155
// NOTE: levelHandler.sortTables() should be called after call addTable calls are done.
156
func (s *levelHandler) addTable(t *table.Table) {
70✔
157
        s.Lock()
70✔
158
        defer s.Unlock()
70✔
159

70✔
160
        s.addSize(t) // Increase totalSize first.
70✔
161
        t.IncrRef()
70✔
162
        s.tables = append(s.tables, t)
70✔
163
}
70✔
164

165
// sortTables sorts tables of levelHandler based on table.Smallest.
166
// Normally it should be called after all addTable calls.
167
func (s *levelHandler) sortTables() {
147✔
168
        s.RLock()
147✔
169
        defer s.RUnlock()
147✔
170

147✔
171
        sort.Slice(s.tables, func(i, j int) bool {
309✔
172
                return y.CompareKeys(s.tables[i].Smallest(), s.tables[j].Smallest()) < 0
162✔
173
        })
162✔
174
}
175

176
func decrRefs(tables []*table.Table) error {
496✔
177
        for _, table := range tables {
1,314✔
178
                if err := table.DecrRef(); err != nil {
818✔
179
                        return err
×
180
                }
×
181
        }
182
        return nil
496✔
183
}
184

185
func newLevelHandler(db *DB, level int) *levelHandler {
2,156✔
186
        return &levelHandler{
2,156✔
187
                level:    level,
2,156✔
188
                strLevel: fmt.Sprintf("l%d", level),
2,156✔
189
                db:       db,
2,156✔
190
        }
2,156✔
191
}
2,156✔
192

193
// tryAddLevel0Table returns true if ok and no stalling.
194
func (s *levelHandler) tryAddLevel0Table(t *table.Table) bool {
288✔
195
        y.AssertTrue(s.level == 0)
288✔
196
        // Need lock as we may be deleting the first table during a level 0 compaction.
288✔
197
        s.Lock()
288✔
198
        defer s.Unlock()
288✔
199
        // Stall (by returning false) if we are above the specified stall setting for L0.
288✔
200
        if len(s.tables) >= s.db.opt.NumLevelZeroTablesStall {
288✔
201
                return false
×
202
        }
×
203

204
        s.tables = append(s.tables, t)
288✔
205
        t.IncrRef()
288✔
206
        s.addSize(t)
288✔
207

288✔
208
        return true
288✔
209
}
210

211
// This should be called while holding the lock on the level.
212
func (s *levelHandler) addSize(t *table.Table) {
859✔
213
        s.totalSize += t.Size()
859✔
214
        s.totalStaleSize += int64(t.StaleDataSize())
859✔
215
}
859✔
216

217
// This should be called while holding the lock on the level.
218
func (s *levelHandler) subtractSize(t *table.Table) {
402✔
219
        s.totalSize -= t.Size()
402✔
220
        s.totalStaleSize -= int64(t.StaleDataSize())
402✔
221
}
402✔
222
func (s *levelHandler) numTables() int {
118,423✔
223
        s.RLock()
118,423✔
224
        defer s.RUnlock()
118,423✔
225
        return len(s.tables)
118,423✔
226
}
118,423✔
227

228
func (s *levelHandler) close() error {
2,093✔
229
        s.RLock()
2,093✔
230
        defer s.RUnlock()
2,093✔
231
        var err error
2,093✔
232
        for _, t := range s.tables {
2,597✔
233
                if closeErr := t.Close(-1); closeErr != nil && err == nil {
504✔
234
                        err = closeErr
×
235
                }
×
236
        }
237
        return y.Wrap(err, "levelHandler.close")
2,093✔
238
}
239

240
// getTableForKey acquires a read-lock to access s.tables. It returns a list of tableHandlers.
241
func (s *levelHandler) getTableForKey(key []byte) ([]*table.Table, func() error) {
4,922,535✔
242
        s.RLock()
4,922,535✔
243
        defer s.RUnlock()
4,922,535✔
244

4,922,535✔
245
        if s.level == 0 {
5,625,773✔
246
                // For level 0, we need to check every table. Remember to make a copy as s.tables may change
703,238✔
247
                // once we exit this function, and we don't want to lock s.tables while seeking in tables.
703,238✔
248
                // CAUTION: Reverse the tables.
703,238✔
249
                out := make([]*table.Table, 0, len(s.tables))
703,238✔
250
                for i := len(s.tables) - 1; i >= 0; i-- {
764,287✔
251
                        out = append(out, s.tables[i])
61,049✔
252
                        s.tables[i].IncrRef()
61,049✔
253
                }
61,049✔
254
                return out, func() error {
1,406,476✔
255
                        for _, t := range out {
764,287✔
256
                                if err := t.DecrRef(); err != nil {
61,049✔
257
                                        return err
×
258
                                }
×
259
                        }
260
                        return nil
703,238✔
261
                }
262
        }
263
        // For level >= 1, we can do a binary search as key range does not overlap.
264
        idx := sort.Search(len(s.tables), func(i int) bool {
4,224,269✔
265
                return y.CompareKeys(s.tables[i].Biggest(), key) >= 0
4,972✔
266
        })
4,972✔
267
        if idx >= len(s.tables) {
8,436,793✔
268
                // Given key is strictly > than every element we have.
4,217,496✔
269
                return nil, func() error { return nil }
8,434,992✔
270
        }
271
        tbl := s.tables[idx]
1,801✔
272
        tbl.IncrRef()
1,801✔
273
        return []*table.Table{tbl}, tbl.DecrRef
1,801✔
274
}
275

276
// get returns value for a given key or the key after that. If not found, return nil.
277
func (s *levelHandler) get(key []byte) (y.ValueStruct, error) {
4,922,535✔
278
        tables, decr := s.getTableForKey(key)
4,922,535✔
279
        keyNoTs := y.ParseKey(key)
4,922,535✔
280

4,922,535✔
281
        hash := y.Hash(keyNoTs)
4,922,535✔
282
        var maxVs y.ValueStruct
4,922,535✔
283
        for _, th := range tables {
4,985,385✔
284
                if th.DoesNotHave(hash) {
67,455✔
285
                        y.NumLSMBloomHitsAdd(s.db.opt.MetricsEnabled, s.strLevel, 1)
4,605✔
286
                        continue
4,605✔
287
                }
288

289
                it := th.NewIterator(0)
58,245✔
290
                defer it.Close()
58,245✔
291

58,245✔
292
                y.NumLSMGetsAdd(s.db.opt.MetricsEnabled, s.strLevel, 1)
58,245✔
293
                it.Seek(key)
58,245✔
294
                if !it.Valid() {
58,286✔
295
                        continue
41✔
296
                }
297
                if y.SameKey(key, it.Key()) {
116,266✔
298
                        if version := y.ParseTs(it.Key()); maxVs.Version < version {
116,117✔
299
                                maxVs = it.ValueCopy()
58,055✔
300
                                maxVs.Version = version
58,055✔
301
                        }
58,055✔
302
                }
303
        }
304
        return maxVs, decr()
4,922,535✔
305
}
306

307
// appendIterators appends iterators to an array of iterators, for merging.
308
// Note: This obtains references for the table handlers. Remember to close these iterators.
309
func (s *levelHandler) appendIterators(iters []y.Iterator, opt *IteratorOptions) []y.Iterator {
287,693✔
310
        s.RLock()
287,693✔
311
        defer s.RUnlock()
287,693✔
312

287,693✔
313
        var topt int
287,693✔
314
        if opt.Reverse {
287,938✔
315
                topt = table.REVERSED
245✔
316
        }
245✔
317
        if s.level == 0 {
328,792✔
318
                // Remember to add in reverse order!
41,099✔
319
                // The newer table at the end of s.tables should be added first as it takes precedence.
41,099✔
320
                // Level 0 tables are not in key sorted order, so we need to consider them one by one.
41,099✔
321
                var out []*table.Table
41,099✔
322
                for _, t := range s.tables {
41,181✔
323
                        if opt.pickTable(t) {
164✔
324
                                out = append(out, t)
82✔
325
                        }
82✔
326
                }
327
                return appendIteratorsReversed(iters, out, topt)
41,099✔
328
        }
329

330
        tables := opt.pickTables(s.tables)
246,594✔
331
        if len(tables) == 0 {
493,073✔
332
                return iters
246,479✔
333
        }
246,479✔
334
        return append(iters, table.NewConcatIterator(tables, topt))
115✔
335
}
336

337
type levelHandlerRLocked struct{}
338

339
// overlappingTables returns the tables that intersect with key range. Returns a half-interval.
340
// This function should already have acquired a read lock, and this is so important the caller must
341
// pass an empty parameter declaring such.
342
func (s *levelHandler) overlappingTables(_ levelHandlerRLocked, kr keyRange) (int, int) {
3,174✔
343
        if len(kr.left) == 0 || len(kr.right) == 0 {
3,174✔
344
                return 0, 0
×
345
        }
×
346
        left := sort.Search(len(s.tables), func(i int) bool {
11,745✔
347
                return y.CompareKeys(kr.left, s.tables[i].Biggest()) <= 0
8,571✔
348
        })
8,571✔
349
        right := sort.Search(len(s.tables), func(i int) bool {
10,977✔
350
                return y.CompareKeys(kr.right, s.tables[i].Smallest()) < 0
7,803✔
351
        })
7,803✔
352
        return left, right
3,174✔
353
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc