• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

vigna / webgraph-rs / 18429616840

11 Oct 2025 12:50PM UTC coverage: 47.997% (+0.03%) from 47.965%
18429616840

push

github

zommiommy
Unlabeled methods

104 of 226 new or added lines in 9 files covered. (46.02%)

21 existing lines in 4 files now uncovered.

4001 of 8336 relevant lines covered (48.0%)

21818046.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.32
/webgraph/src/utils/batch_codec/grouped_gaps.rs
1
/*
2
 * SPDX-FileCopyrightText: 2025 Tommaso Fontana
3
 *
4
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
5
 */
6

7
use super::{BitReader, BitWriter};
8
use crate::traits::SortedIterator;
9
use crate::utils::{ArcMmapHelper, MmapHelper, Triple};
10
use crate::{
11
    traits::{BitDeserializer, BitSerializer},
12
    utils::BatchCodec,
13
};
14

15
use std::sync::Arc;
16

17
use anyhow::{Context, Result};
18
use dsi_bitstream::prelude::*;
19
use mmap_rs::MmapFlags;
20
use rdst::*;
21

22
#[derive(Clone, Debug, Default)]
23
/// A codec for encoding and decoding batches of triples using grouped gap compression.
24
///
25
/// This codec encodes triples of the form `(src, dst, label)` by grouping edges with the same source node,
26
/// and encoding the gaps between consecutive sources and destinations using a specified code (default: gamma).
27
/// The outdegree (number of edges for each source) is also encoded using the specified code.
28
///
29
/// ## Type Parameters
30
/// - `S`: Serializer for the labels, implementing [`BitSerializer`] for the label type.
31
/// - `D`: Deserializer for the labels, implementing [`BitDeserializer`] for the label type.
32
/// - `OUTDEGREE_CODE`: Code used for encoding outdegrees (default: gamma).
33
/// - `SRC_CODE`: Code used for encoding source gaps (default: gamma).
34
/// - `DST_CODE`: Code used for encoding destination gaps (default: gamma).
35
///
36
/// ## Fields
37
/// - `serializer`: The label serializer.
38
/// - `deserializer`: The label deserializer.
39
///
40
/// ## Encoding Format
41
/// 1. The batch length is written using delta coding.
42
/// 2. For each group of triples with the same source:
43
///     - The gap from the previous source is encoded.
44
///     - The outdegree (number of edges for this source) is encoded.
45
///     - For each destination:
46
///         - The gap from the previous destination is encoded.
47
///         - The label is serialized.
48
///
49
/// The bit deserializer must be [`Clone`] because we need one for each
50
/// [`GroupedGapsIterator`], and there are possible scenarios in which the
51
/// deserializer might be stateful.
52
///
53
/// ## Choosing the codes
54
///
55
/// When transposing `enwiki-2024`, these are the top 10 codes for src gaps, outdegree, and dst gaps:
56
/// ```ignore
57
/// Outdegree stats
58
///   Code: ExpGolomb(3) Size: 34004796
59
///   Code: ExpGolomb(2) Size: 34101784
60
///   Code: ExpGolomb(4) Size: 36036394
61
///   Code: Zeta(2)      Size: 36231582
62
///   Code: ExpGolomb(1) Size: 36369750
63
///   Code: Zeta(3)      Size: 36893285
64
///   Code: Pi(2)        Size: 37415701
65
///   Code: Zeta(4)      Size: 38905267
66
///   Code: Golomb(20)   Size: 38963840
67
///   Code: Golomb(19)   Size: 39118201
68
/// Src stats
69
///   Code: Golomb(2)    Size: 12929998
70
///   Code: Rice(1)      Size: 12929998
71
///   Code: Unary        Size: 13025332
72
///   Code: Golomb(1)    Size: 13025332
73
///   Code: Rice(0)      Size: 13025332
74
///   Code: ExpGolomb(1) Size: 13319930
75
///   Code: Golomb(4)    Size: 18732384
76
///   Code: Rice(2)      Size: 18732384
77
///   Code: Golomb(3)    Size: 18736573
78
///   Code: ExpGolomb(2) Size: 18746122
79
/// Dst stats
80
///   Code: Pi(2)   Size: 2063880685
81
///   Code: Pi(3)   Size: 2074138948
82
///   Code: Zeta(3) Size: 2122730298
83
///   Code: Zeta(4) Size: 2123948774
84
///   Code: Zeta(5) Size: 2169131998
85
///   Code: Pi(4)   Size: 2176097847
86
///   Code: Zeta(2) Size: 2226573622
87
///   Code: Zeta(6) Size: 2237680403
88
///   Code: Delta   Size: 2272691460
89
///   Code: Zeta(7) Size: 2305354857
90
/// ```
91
///
92
/// The best codes are `Golomb(2)` for src gaps, `ExpGolomb(3)` for outdegree, and `Pi(2)` for dst gaps.
93
/// However, `Golomb` can perform poorly if the data don't follow the expected distribution,
94
/// so the recommended defaults are `Gamma` for src gaps, `ExpGolomb3` for outdegree, and `Delta` for dst gaps,
95
/// as they are universal codes.
96
pub struct GroupedGapsCodec<
97
    S: BitSerializer<NE, BitWriter> = (),
98
    D: BitDeserializer<NE, BitReader, DeserType = S::SerType> + Clone = (),
99
    const OUTDEGREE_CODE: usize = { dsi_bitstream::dispatch::code_consts::EXP_GOLOMB3 },
100
    const SRC_CODE: usize = { dsi_bitstream::dispatch::code_consts::GAMMA },
101
    const DST_CODE: usize = { dsi_bitstream::dispatch::code_consts::DELTA },
102
> {
103
    /// Serializer for the labels
104
    pub serializer: S,
105
    /// Deserializer for the labels
106
    pub deserializer: D,
107
}
108

109
impl<S, D, const OUTDEGREE_CODE: usize, const SRC_CODE: usize, const DST_CODE: usize> BatchCodec
110
    for GroupedGapsCodec<S, D, OUTDEGREE_CODE, SRC_CODE, DST_CODE>
111
where
112
    S: BitSerializer<NE, BitWriter> + Send + Sync,
113
    D: BitDeserializer<NE, BitReader, DeserType = S::SerType> + Send + Sync + Clone,
114
    S::SerType: Send + Sync + Copy + 'static, // needed by radix sort
115
{
116
    type Label = S::SerType;
117
    type DecodedBatch = GroupedGapsIterator<D, OUTDEGREE_CODE, SRC_CODE, DST_CODE>;
118

119
    fn encode_batch(
432✔
120
        &self,
121
        path: impl AsRef<std::path::Path>,
122
        batch: &mut [(usize, usize, Self::Label)],
123
    ) -> Result<usize> {
124
        let start = std::time::Instant::now();
864✔
125
        Triple::cast_batch_mut(batch).radix_sort_unstable();
864✔
126
        log::debug!("Sorted {} arcs in {:?}", batch.len(), start.elapsed());
432✔
127
        self.encode_sorted_batch(path, batch)
1,728✔
128
    }
129

130
    fn encode_sorted_batch(
432✔
131
        &self,
132
        path: impl AsRef<std::path::Path>,
133
        batch: &[(usize, usize, Self::Label)],
134
    ) -> Result<usize> {
135
        debug_assert!(Triple::cast_batch(batch).is_sorted(), "Batch is not sorted");
1,296✔
136
        // create a batch file where to dump
137
        let file_path = path.as_ref();
1,296✔
138
        let file = std::io::BufWriter::with_capacity(
139
            1 << 16,
432✔
140
            std::fs::File::create(file_path).with_context(|| {
1,296✔
NEW
141
                format!(
×
NEW
142
                    "Could not create BatchIterator temporary file {}",
×
NEW
143
                    file_path.display()
×
144
                )
145
            })?,
146
        );
147
        // create a bitstream to write to the file
NEW
148
        let mut stream = <BufBitWriter<NE, _>>::new(<WordAdapter<usize, _>>::new(file));
×
149

150
        // prefix the stream with the length of the batch
151
        // we use a delta code since it'll be a big number most of the time
NEW
152
        stream
×
NEW
153
            .write_delta(batch.len() as u64)
×
154
            .context("Could not write length")?;
155

156
        // dump the triples to the bitstream
157
        let mut prev_src = 0;
432✔
NEW
158
        let mut written_bits = 0;
×
NEW
159
        let mut i = 0;
×
160
        while i < batch.len() {
4,726,354✔
161
            let (src, _, _) = batch[i];
4,725,490✔
162
            // write the source gap as gamma
163
            written_bits += ConstCode::<SRC_CODE>
2,362,745✔
164
                .write(&mut stream, (src - prev_src) as _)
7,088,235✔
165
                .with_context(|| format!("Could not write {src} after {prev_src}"))?;
2,362,745✔
166
            // figure out how many edges have this source
167
            let outdegree = batch[i..].iter().take_while(|t| t.0 == src).count();
84,259,241✔
168
            // write the outdegree
NEW
169
            written_bits += ConstCode::<OUTDEGREE_CODE>
×
NEW
170
                .write(&mut stream, outdegree as _)
×
NEW
171
                .with_context(|| format!("Could not write outdegree {outdegree} for {src}"))?;
×
172

173
            // encode the destinations
174
            let mut prev_dst = 0;
2,362,745✔
NEW
175
            for _ in 0..outdegree {
×
176
                let (_, dst, label) = &batch[i];
38,585,935✔
177
                // write the destination gap as gamma
NEW
178
                written_bits += ConstCode::<DST_CODE>
×
NEW
179
                    .write(&mut stream, (dst - prev_dst) as _)
×
NEW
180
                    .with_context(|| format!("Could not write {dst} after {prev_dst}"))?;
×
181
                // write the label
182
                written_bits += self
38,585,935✔
183
                    .serializer
38,585,935✔
NEW
184
                    .serialize(label, &mut stream)
×
NEW
185
                    .context("Could not serialize label")?;
×
186
                prev_dst = *dst;
38,585,935✔
NEW
187
                i += 1;
×
188
            }
189
            prev_src = src;
2,362,745✔
190
        }
191
        // flush the stream and reset the buffer
192
        written_bits += stream.flush().context("Could not flush stream")?;
432✔
193

194
        Ok(written_bits)
432✔
195
    }
196

197
    fn decode_batch(&self, path: impl AsRef<std::path::Path>) -> Result<Self::DecodedBatch> {
141✔
198
        // open the file
199
        let mut stream = <BufBitReader<NE, _>>::new(MemWordReader::new(ArcMmapHelper(Arc::new(
141✔
200
            MmapHelper::mmap(
141✔
201
                path.as_ref(),
282✔
202
                MmapFlags::TRANSPARENT_HUGE_PAGES | MmapFlags::SEQUENTIAL,
141✔
203
            )
204
            .with_context(|| format!("Could not mmap {}", path.as_ref().display()))?,
141✔
205
        ))));
206

207
        // read the length of the batch (first value in the stream)
208
        let len = stream.read_delta().context("Could not read length")? as usize;
141✔
209

210
        // create the iterator
NEW
211
        Ok(GroupedGapsIterator {
×
NEW
212
            deserializer: self.deserializer.clone(),
×
NEW
213
            stream,
×
NEW
214
            len,
×
NEW
215
            current: 0,
×
NEW
216
            src: 0,
×
NEW
217
            dst_left: 0,
×
NEW
218
            prev_dst: 0,
×
219
        })
220
    }
221
}
222

223
#[derive(Clone, Debug)]
224
/// An iterator over triples encoded with gaps, this is returned by [`GroupedGapsCodec`].
225
pub struct GroupedGapsIterator<
226
    D: BitDeserializer<NE, BitReader> = (),
227
    const OUTDEGREE_CODE: usize = { dsi_bitstream::dispatch::code_consts::GAMMA },
228
    const SRC_CODE: usize = { dsi_bitstream::dispatch::code_consts::GAMMA },
229
    const DST_CODE: usize = { dsi_bitstream::dispatch::code_consts::GAMMA },
230
> {
231
    /// Deserializer for the labels
232
    deserializer: D,
233
    /// Bitstream to read from
234
    stream: BitReader,
235
    /// Length of the iterator (number of triples)
236
    len: usize,
237
    /// Current position in the iterator
238
    current: usize,
239
    /// Current source node
240
    src: usize,
241
    /// Number of destinations left for the current source
242
    dst_left: usize,
243
    /// Previous destination node
244
    prev_dst: usize,
245
}
246

247
unsafe impl<
248
        D: BitDeserializer<NE, BitReader>,
249
        const OUTDEGREE_CODE: usize,
250
        const SRC_CODE: usize,
251
        const DST_CODE: usize,
252
    > SortedIterator for GroupedGapsIterator<D, OUTDEGREE_CODE, SRC_CODE, DST_CODE>
253
{
254
}
255

256
impl<
257
        D: BitDeserializer<NE, BitReader>,
258
        const OUTDEGREE_CODE: usize,
259
        const SRC_CODE: usize,
260
        const DST_CODE: usize,
261
    > Iterator for GroupedGapsIterator<D, OUTDEGREE_CODE, SRC_CODE, DST_CODE>
262
{
263
    type Item = (usize, usize, D::DeserType);
264
    fn next(&mut self) -> Option<Self::Item> {
67,777,806✔
265
        if self.current >= self.len {
67,777,806✔
266
            return None;
130✔
267
        }
NEW
268
        if self.dst_left == 0 {
×
269
            // read a new source
270
            let src_gap = ConstCode::<SRC_CODE>.read(&mut self.stream).ok()?;
20,320,970✔
NEW
271
            self.src += src_gap as usize;
×
272
            // read the outdegree
NEW
273
            self.dst_left = ConstCode::<OUTDEGREE_CODE>.read(&mut self.stream).ok()? as usize;
×
274
            self.prev_dst = 0;
4,064,194✔
275
        }
276

277
        let dst_gap = ConstCode::<DST_CODE>.read(&mut self.stream).ok()?;
135,555,352✔
278
        let label = self.deserializer.deserialize(&mut self.stream).ok()?;
67,777,676✔
NEW
279
        self.prev_dst += dst_gap as usize;
×
NEW
280
        self.current += 1;
×
NEW
281
        self.dst_left -= 1;
×
NEW
282
        Some((self.src, self.prev_dst, label))
×
283
    }
284

NEW
285
    fn size_hint(&self) -> (usize, Option<usize>) {
×
NEW
286
        (self.len(), Some(self.len()))
×
287
    }
288
}
289

290
impl<
291
        D: BitDeserializer<NE, BitReader>,
292
        const OUTDEGREE_CODE: usize,
293
        const SRC_CODE: usize,
294
        const DST_CODE: usize,
295
    > ExactSizeIterator for GroupedGapsIterator<D, OUTDEGREE_CODE, SRC_CODE, DST_CODE>
296
{
NEW
297
    fn len(&self) -> usize {
×
NEW
298
        self.len - self.current
×
299
    }
300
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc