• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

vigna / webgraph-rs / 19076069272

04 Nov 2025 04:42PM UTC coverage: 61.785% (-0.2%) from 61.976%
19076069272

push

github

vigna
Fixed doctests

5143 of 8324 relevant lines covered (61.79%)

30278823.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.06
/webgraph/src/utils/mod.rs
1
/*
2
 * SPDX-FileCopyrightText: 2023 Inria
3
 * SPDX-FileCopyrightText: 2023 Sebastiano Vigna
4
 *
5
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6
 */
7

8
//! Miscellaneous utilities.
9

10
use rand::Rng;
11
use std::path::PathBuf;
12

13
/// Creates a new random dir inside the given folder
14
pub fn temp_dir<P: AsRef<std::path::Path>>(base: P) -> anyhow::Result<PathBuf> {
9✔
15
    let mut base = base.as_ref().to_owned();
27✔
16
    const ALPHABET: &[u8] = b"0123456789abcdef";
17
    let mut rnd = rand::rng();
18✔
18
    let mut random_str = String::new();
18✔
19
    loop {
×
20
        random_str.clear();
18✔
21
        for _ in 0..16 {
297✔
22
            let idx = rnd.random_range(0..ALPHABET.len());
720✔
23
            random_str.push(ALPHABET[idx] as char);
432✔
24
        }
25
        base.push(&random_str);
27✔
26

27
        if !base.exists() {
9✔
28
            std::fs::create_dir(&base)?;
18✔
29
            return Ok(base);
9✔
30
        }
31
        base.pop();
×
32
    }
33
}
34

35
mod circular_buffer;
36
pub(crate) use circular_buffer::*;
37

38
mod mmap_helper;
39
pub use mmap_helper::*;
40

41
mod java_perm;
42
pub use java_perm::*;
43

44
mod granularity;
45
pub use granularity::*;
46

47
pub mod sort_pairs;
48
pub use sort_pairs::SortPairs;
49

50
pub mod par_sort_pairs;
51
pub use par_sort_pairs::ParSortPairs;
52

53
pub mod par_sort_iters;
54
pub use par_sort_iters::ParSortIters;
55

56
use crate::graphs::{
57
    arc_list_graph::Iter,
58
    bvgraph::{Decode, Encode},
59
};
60

61
/// A decoder that encodes the read values using the given encoder.
62
/// This is commonly used to change the codes of a graph without decoding and
63
/// re-encoding it but by changing the codes.
64
pub struct Converter<D: Decode, E: Encode> {
65
    pub decoder: D,
66
    pub encoder: E,
67
    pub offset: usize,
68
}
69

70
impl<D: Decode, E: Encode> Decode for Converter<D, E> {
71
    // TODO: implement correctly start_node/end_node
72
    #[inline(always)]
73
    fn read_outdegree(&mut self) -> u64 {
×
74
        let res = self.decoder.read_outdegree();
×
75
        self.offset += self.encoder.write_outdegree(res).unwrap();
×
76
        res
×
77
    }
78
    #[inline(always)]
79
    fn read_reference_offset(&mut self) -> u64 {
×
80
        let res = self.decoder.read_reference_offset();
×
81
        self.offset += self.encoder.write_reference_offset(res).unwrap();
×
82
        res
×
83
    }
84
    #[inline(always)]
85
    fn read_block_count(&mut self) -> u64 {
×
86
        let res = self.decoder.read_block_count();
×
87
        self.offset += self.encoder.write_block_count(res).unwrap();
×
88
        res
×
89
    }
90
    #[inline(always)]
91
    fn read_block(&mut self) -> u64 {
×
92
        let res = self.decoder.read_block();
×
93
        self.offset += self.encoder.write_block(res).unwrap();
×
94
        res
×
95
    }
96
    #[inline(always)]
97
    fn read_interval_count(&mut self) -> u64 {
×
98
        let res = self.decoder.read_interval_count();
×
99
        self.offset += self.encoder.write_interval_count(res).unwrap();
×
100
        res
×
101
    }
102
    #[inline(always)]
103
    fn read_interval_start(&mut self) -> u64 {
×
104
        let res = self.decoder.read_interval_start();
×
105
        self.offset += self.encoder.write_interval_start(res).unwrap();
×
106
        res
×
107
    }
108
    #[inline(always)]
109
    fn read_interval_len(&mut self) -> u64 {
×
110
        let res = self.decoder.read_interval_len();
×
111
        self.offset += self.encoder.write_interval_len(res).unwrap();
×
112
        res
×
113
    }
114
    #[inline(always)]
115
    fn read_first_residual(&mut self) -> u64 {
×
116
        let res = self.decoder.read_first_residual();
×
117
        self.offset += self.encoder.write_first_residual(res).unwrap();
×
118
        res
×
119
    }
120
    #[inline(always)]
121
    fn read_residual(&mut self) -> u64 {
×
122
        let res = self.decoder.read_residual();
×
123
        self.offset += self.encoder.write_residual(res).unwrap();
×
124
        res
×
125
    }
126
}
127

128
/// An enum expressing the memory requirements for batched algorithms
129
/// such as [`SortPairs`], [`ParSortPairs`], and [`ParSortIters`].
130
///
131
/// This type implements [`Mul`](core::ops::Mul) and [`Div`](core::ops::Div) to
132
/// scale the memory usage requirements by a given factor, independently of the
133
/// variant.
134
#[derive(Clone, Copy, Debug)]
135
pub enum MemoryUsage {
136
    /// The target overall memory usage in bytes.
137
    ///
138
    /// This is the more user-friendly option. The actual number of elements
139
    /// can be computed using [`batch_size`](MemoryUsage::batch_size).
140
    MemorySize(usize),
141
    /// The number of elements used in all batches.
142
    ///
143
    /// This is a more low-level option that gives more control to the user, but
144
    /// the actual memory usage will depend on the size of labels (if any).
145
    BatchSize(usize),
146
}
147

148
/// Default implementation, returning half of the physical RAM.
149
impl Default for MemoryUsage {
150
    fn default() -> Self {
16✔
151
        Self::from_perc(50.0)
16✔
152
    }
153
}
154

155
impl core::fmt::Display for MemoryUsage {
156
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4✔
157
        match self {
4✔
158
            MemoryUsage::MemorySize(size) => write!(f, "{} bytes", size),
16✔
159
            MemoryUsage::BatchSize(size) => write!(f, "{} elements", size),
×
160
        }
161
    }
162
}
163

164
impl core::ops::Mul<usize> for MemoryUsage {
165
    type Output = MemoryUsage;
166

167
    fn mul(self, rhs: usize) -> Self::Output {
×
168
        match self {
×
169
            MemoryUsage::MemorySize(size) => MemoryUsage::MemorySize(size * rhs),
×
170
            MemoryUsage::BatchSize(size) => MemoryUsage::BatchSize(size * rhs),
×
171
        }
172
    }
173
}
174

175
impl core::ops::Div<usize> for MemoryUsage {
176
    type Output = MemoryUsage;
177

178
    fn div(self, rhs: usize) -> Self::Output {
16✔
179
        match self {
16✔
180
            MemoryUsage::MemorySize(size) => MemoryUsage::MemorySize(size / rhs),
32✔
181
            MemoryUsage::BatchSize(size) => MemoryUsage::BatchSize(size / rhs),
×
182
        }
183
    }
184
}
185

186
impl MemoryUsage {
187
    /// Creates a new memory usage expressed as a percentage of the
188
    /// physical RAM.
189
    pub fn from_perc(perc: f64) -> Self {
24✔
190
        let system = sysinfo::System::new_with_specifics(
191
            sysinfo::RefreshKind::nothing()
24✔
192
                .with_memory(sysinfo::MemoryRefreshKind::nothing().with_ram()),
72✔
193
        );
194
        MemoryUsage::MemorySize(
195
            usize::try_from((system.total_memory() as f64 * perc / 100.0) as u64)
72✔
196
                .expect("System memory overflows usize"),
24✔
197
        )
198
    }
199

200
    /// Returns the batch size for elements of type `T`.
201
    ///
202
    /// If the [memory usage is expressed as a number of
203
    /// bytes](MemoryUsage::MemorySize), this method divides the number of bytes
204
    /// by the size of `T` to obtain the number of elements. Otherwise, [it just
205
    /// returns specified batch size](MemoryUsage::BatchSize).
206
    pub fn batch_size<T>(&self) -> usize {
135✔
207
        match &self {
135✔
208
            MemoryUsage::MemorySize(memory_size) => {
35✔
209
                let elem_size = std::mem::size_of::<T>();
70✔
210
                assert!(elem_size > 0, "Element size cannot be zero");
70✔
211
                memory_size / elem_size
35✔
212
            }
213
            MemoryUsage::BatchSize(batch_size) => *batch_size,
200✔
214
        }
215
    }
216
}
217

218
/// A structure holding partition iterators and their boundaries.
219
///
220
/// This type holds a list of iterators and a list of boundaries, one more
221
/// than the number of iterators. The implied semantics is that each iterator
222
/// returns (labelled) pairs of nodes, and that the first element of
223
/// each pair sits between the boundaries associated with the iterator.
224
///
225
/// This structures is returned by [`ParSortPairs`] and [`ParSortIters`] and can
226
/// easily be converted into lenders for use with
227
/// [`BvComp::parallel_iter`](crate::graphs::bvgraph::BvComp::parallel_iter)
228
/// using a convenient implementation of the [`From`] trait.
229
///
230
/// Note that it sufficient to write `let lenders: Vec<_> = split_iters.into()`
231
/// to perform the conversion. Type inference might not work properly if the
232
/// call is embedded in a larger expression, in which case an explicit type
233
/// annotation might be necessary.
234
pub struct SplitIters<I> {
235
    pub boundaries: Box<[usize]>,
236
    pub iters: Box<[I]>,
237
}
238

239
impl<I> SplitIters<I> {
240
    pub fn new(boundaries: Box<[usize]>, iters: Box<[I]>) -> Self {
2✔
241
        Self { boundaries, iters }
242
    }
243
}
244

245
impl<I> From<(Box<[usize]>, Box<[I]>)> for SplitIters<I> {
246
    fn from((boundaries, iters): (Box<[usize]>, Box<[I]>)) -> Self {
×
247
        Self::new(boundaries, iters)
×
248
    }
249
}
250

251
/// Conversion of a [`SplitIters`] of iterators on unlabeled pairs into a
252
/// sequence of pairs of starting points and associated lenders.
253
///
254
/// This is useful for converting the output of sorting utilities like
255
/// [`ParSortPairs`] or [`ParSortIters`] into a form suitable for
256
/// [`BvComp::parallel_iter`](crate::graphs::bvgraph::BvComp::parallel_iter)
257
/// when working with unlabeled graphs.
258
///
259
/// The pairs `(src, dst)` are automatically converted to labeled form with unit
260
/// labels, and the resulting lenders are wrapped with
261
/// [`LeftIterator`](crate::labels::proj::LeftIterator) to project out just the
262
/// successor nodes.
263
///
264
/// Note that it sufficient to write `let lenders: Vec<_> = split_iters.into()`
265
/// to perform the conversion. Type inference might not work properly if the
266
/// call is embedded in a larger expression, in which case an explicit type
267
/// annotation might be necessary.
268
impl<
269
        I: Iterator<Item = (usize, usize)> + Send + Sync,
270
        IT: IntoIterator<Item = (usize, usize), IntoIter = I>,
271
    > From<SplitIters<IT>>
272
    for Vec<
273
        crate::labels::proj::LeftIterator<
274
            Iter<(), std::iter::Map<I, fn((usize, usize)) -> ((usize, usize), ())>>,
275
        >,
276
    >
277
{
278
    fn from(split: SplitIters<IT>) -> Self {
1✔
279
        Box::into_iter(split.iters)
2✔
280
            .enumerate()
281
            .map(|(i, iter)| {
5✔
282
                let start_node = split.boundaries[i];
8✔
283
                let end_node = split.boundaries[i + 1];
8✔
284
                let num_partition_nodes = end_node - start_node;
8✔
285
                // Map pairs to labeled pairs with unit labels
286
                let map_fn: fn((usize, usize)) -> ((usize, usize), ()) = |pair| (pair, ());
20✔
287
                let labeled_iter = iter.into_iter().map(map_fn);
20✔
288
                let lender = Iter::try_new_from(num_partition_nodes, labeled_iter, start_node)
20✔
289
                    .expect("Iterator should start from the expected first node");
8✔
290
                // Wrap with LeftIterator to project out just the successor
291
                crate::labels::proj::LeftIterator(lender)
4✔
292
            })
293
            .collect()
294
    }
295
}
296

297
/// Conversion of a [`SplitIters`] of iterators on labelled pairs into a
298
/// sequences of pairs of starting points and associated lenders.
299
///
300
/// This is useful for converting the output of sorting utilities like
301
/// [`ParSortPairs`] or [`ParSortIters`] into a form suitable for
302
/// [`BvComp::parallel_iter`](crate::graphs::bvgraph::BvComp::parallel_iter).
303
///
304
/// Note that it sufficient to write `let lenders: Vec<_> = split_iters.into()`
305
/// to perform the conversion. Type inference might not work properly if the
306
/// call is embedded in a larger expression, in which case an explicit type
307
/// annotation might be necessary.
308
impl<
309
        L: Clone + Copy + 'static,
310
        I: Iterator<Item = ((usize, usize), L)> + Send + Sync,
311
        IT: IntoIterator<Item = ((usize, usize), L), IntoIter = I>,
312
    > From<SplitIters<IT>> for Vec<Iter<L, I>>
313
{
314
    fn from(split: SplitIters<IT>) -> Self {
1✔
315
        Box::into_iter(split.iters)
2✔
316
            .enumerate()
317
            .map(|(i, iter)| {
4✔
318
                let start_node = split.boundaries[i];
6✔
319
                let end_node = split.boundaries[i + 1];
6✔
320
                let num_partition_nodes = end_node - start_node;
6✔
321

322
                Iter::try_new_from(num_partition_nodes, iter.into_iter(), start_node)
15✔
323
                    .expect("Iterator should start from the expected first node")
6✔
324
            })
325
            .collect()
326
    }
327
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc