15788522925

Committed 20 Jun 2025 09:59PM UTC coverage: 50.317% (+0.07%) from 50.251%

Build # 15788522925

Build Type

push

github

Committed by

vigna

Commit Message

Docs

Run Details

3805 of 7562 relevant lines covered (50.32%)

23808941.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.55

/webgraph/src/graphs/bvgraph/sequential.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Sebastiano Vigna
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use std::path::PathBuf;

use super::*;
use crate::utils::CircularBuffer;
use anyhow::Result;
use bitflags::Flags;
use dsi_bitstream::codes::ToInt;
use dsi_bitstream::traits::BitSeek;
use dsi_bitstream::traits::BE;
use lender::*;

/// A sequential BvGraph that can be read from a `codes_reader_builder`.
/// The builder is needed because we should be able to create multiple iterators
/// and this allows us to have a single place where to store the mmapped file.
#[derive(Debug, Clone)]
pub struct BvGraphSeq<F> {
    factory: F,
    number_of_nodes: usize,
    number_of_arcs: Option<u64>,
    compression_window: usize,
    min_interval_length: usize,
}

impl BvGraphSeq<()> {
    pub fn with_basename(
        basename: impl AsRef<std::path::Path>,
    ) -> LoadConfig<BE, Sequential, Dynamic, Mmap, Mmap> {
        LoadConfig {
            basename: PathBuf::from(basename.as_ref()),
            graph_load_flags: Flags::empty(),
            offsets_load_flags: Flags::empty(),
            _marker: std::marker::PhantomData,
        }
    }
}

impl<F: SequentialDecoderFactory> SplitLabeling for BvGraphSeq<F>
where
    for<'a> <F as SequentialDecoderFactory>::Decoder<'a>: Clone + Send + Sync,
{
    type SplitLender<'a>
        = split::seq::Lender<'a, BvGraphSeq<F>>
    where
        Self: 'a;
    type IntoIterator<'a>
        = split::seq::IntoIterator<'a, BvGraphSeq<F>>
    where
        Self: 'a;

    fn split_iter(&self, how_many: usize) -> Self::IntoIterator<'_> {
        split::seq::Iter::new(self.iter(), self.num_nodes(), how_many)
    }
}

impl<F: SequentialDecoderFactory> SequentialLabeling for BvGraphSeq<F> {
    type Label = usize;
    type Lender<'a>
        = Iter<F::Decoder<'a>>
    where
        Self: 'a;

    #[inline(always)]
    /// Returns the number of nodes in the graph
    fn num_nodes(&self) -> usize {
        self.number_of_nodes
    }

    #[inline(always)]
    fn num_arcs_hint(&self) -> Option<u64> {
        self.number_of_arcs
    }

    #[inline(always)]
    fn iter_from(&self, from: usize) -> Self::Lender<'_> {
        let mut iter = Iter::new(
            self.factory.new_decoder().unwrap(),
            self.number_of_nodes,
            self.compression_window,
            self.min_interval_length,
        );

        let _ = iter.advance_by(from);

        iter
    }
}

impl<F: SequentialDecoderFactory> SequentialGraph for BvGraphSeq<F> {}

impl<'a, F: SequentialDecoderFactory> IntoLender for &'a BvGraphSeq<F> {
    type Lender = <BvGraphSeq<F> as SequentialLabeling>::Lender<'a>;

    #[inline(always)]
    fn into_lender(self) -> Self::Lender {
        self.iter()
    }
}

impl<F: SequentialDecoderFactory> BvGraphSeq<F> {
    /// Creates a new sequential graph from a codes reader builder
    /// and the number of nodes.
    pub fn new(
        codes_reader_builder: F,
        number_of_nodes: usize,
        number_of_arcs: Option<u64>,
        compression_window: usize,
        min_interval_length: usize,
    ) -> Self {
        Self {
            factory: codes_reader_builder,
            number_of_nodes,
            number_of_arcs,
            compression_window,
            min_interval_length,
        }
    }

    #[inline(always)]
    pub fn map_factory<F1, F0>(self, map_func: F0) -> BvGraphSeq<F1>
    where
        F0: FnOnce(F) -> F1,
        F1: SequentialDecoderFactory,
    {
        BvGraphSeq {
            factory: map_func(self.factory),
            number_of_nodes: self.number_of_nodes,
            number_of_arcs: self.number_of_arcs,
            compression_window: self.compression_window,
            min_interval_length: self.min_interval_length,
        }
    }

    #[inline(always)]
    /// Consume self and return the factory
    pub fn into_inner(self) -> F {
        self.factory
    }
}

impl<F: SequentialDecoderFactory> BvGraphSeq<F>
where
    for<'a> F::Decoder<'a>: Decode,
{
    #[inline(always)]
    /// Creates an iterator specialized in the degrees of the nodes.
    /// This is slightly faster because it can avoid decoding some of the nodes
    /// and completely skip the merging step.
    pub fn offset_deg_iter(&self) -> OffsetDegIter<F::Decoder<'_>> {
        OffsetDegIter::new(
            self.factory.new_decoder().unwrap(),
            self.number_of_nodes,
            self.compression_window,
            self.min_interval_length,
        )
    }
}

/// A fast sequential iterator over the nodes of the graph and their successors.
/// This iterator does not require to know the offsets of each node in the graph.
#[derive(Debug, Clone)]
pub struct Iter<D: Decode> {
    pub(crate) number_of_nodes: usize,
    pub(crate) compression_window: usize,
    pub(crate) min_interval_length: usize,
    pub(crate) decoder: D,
    pub(crate) backrefs: CircularBuffer<Vec<usize>>,
    pub(crate) current_node: usize,
}

impl<D: Decode + BitSeek> Iter<D> {
    #[inline(always)]
    /// Forward the call of `get_pos` to the inner `codes_reader`.
    /// This returns the current bits offset in the bitstream.
    pub fn bit_pos(&mut self) -> Result<u64, <D as BitSeek>::Error> {
        self.decoder.bit_pos()
    }
}

impl<D: Decode> Iter<D> {
    /// Creates a new iterator from a codes reader
    pub fn new(
        decoder: D,
        number_of_nodes: usize,
        compression_window: usize,
        min_interval_length: usize,
    ) -> Self {
        Self {
            number_of_nodes,
            compression_window,
            min_interval_length,
            decoder,
            backrefs: CircularBuffer::new(compression_window + 1),
            current_node: 0,
        }
    }

    /// Get the successors of the next node in the stream
    pub fn next_successors(&mut self) -> Result<&[usize]> {
        let mut res = self.backrefs.take(self.current_node);
        res.clear();
        self.get_successors_iter_priv(self.current_node, &mut res)?;
        let res = self.backrefs.replace(self.current_node, res);
        self.current_node += 1;
        Ok(res)
    }

    #[inline(always)]
    /// Inner method called by `next_successors` and the iterator `next` method
    fn get_successors_iter_priv(&mut self, node_id: usize, results: &mut Vec<usize>) -> Result<()> {
        let degree = self.decoder.read_outdegree() as usize;
        // no edges, we are done!
        if degree == 0 {
            return Ok(());
        }

        // ensure that we have enough capacity in the vector for not reallocating
        results.reserve(degree.saturating_sub(results.capacity()));
        // read the reference offset
        let ref_delta = if self.compression_window != 0 {
            self.decoder.read_reference_offset() as usize
        } else {
            0
        };
        // if we copy nodes from a previous one
        if ref_delta != 0 {
            // compute the node id of the reference
            let reference_node_id = node_id - ref_delta;
            // retrieve the data
            let neighbours = &self.backrefs[reference_node_id];
            //debug_assert!(!neighbours.is_empty());
            // get the info on which destinations to copy
            let number_of_blocks = self.decoder.read_block_count() as usize;
            // no blocks, we copy everything
            if number_of_blocks == 0 {
                results.extend_from_slice(neighbours);
            } else {
                // otherwise we copy only the blocks of even index
                // the first block could be zero
                let mut idx = self.decoder.read_block() as usize;
                results.extend_from_slice(&neighbours[..idx]);

                // while the other can't
                for block_id in 1..number_of_blocks {
                    let block = self.decoder.read_block() as usize;
                    let end = idx + block + 1;
                    if block_id % 2 == 0 {
                        results.extend_from_slice(&neighbours[idx..end]);
                    }
                    idx = end;
                }
                if number_of_blocks & 1 == 0 {
                    results.extend_from_slice(&neighbours[idx..]);
                }
            }
        };

        // if we still have to read nodes
        let nodes_left_to_decode = degree - results.len();
        if nodes_left_to_decode != 0 && self.min_interval_length != 0 {
            // read the number of intervals
            let number_of_intervals = self.decoder.read_interval_count() as usize;
            if number_of_intervals != 0 {
                // pre-allocate with capacity for efficiency
                let node_id_offset = self.decoder.read_interval_start().to_int();
                let mut start = (node_id as i64 + node_id_offset) as usize;
                let mut delta = self.decoder.read_interval_len() as usize;
                delta += self.min_interval_length;
                // save the first interval
                results.extend(start..(start + delta));
                start += delta;
                // decode the intervals
                for _ in 1..number_of_intervals {
                    start += 1 + self.decoder.read_interval_start() as usize;
                    delta = self.decoder.read_interval_len() as usize;
                    delta += self.min_interval_length;

                    results.extend(start..(start + delta));

                    start += delta;
                }
            }
        }

        // decode the extra nodes if needed
        let nodes_left_to_decode = degree - results.len();
        if nodes_left_to_decode != 0 {
            // pre-allocate with capacity for efficiency
            let node_id_offset = self.decoder.read_first_residual().to_int();
            let mut extra = (node_id as i64 + node_id_offset) as usize;
            results.push(extra);
            // decode the successive extra nodes
            for _ in 1..nodes_left_to_decode {
                extra += 1 + self.decoder.read_residual() as usize;
                results.push(extra);
            }
        }

        results.sort();
        Ok(())
    }
}

impl<'succ, D: Decode> NodeLabelsLender<'succ> for Iter<D> {
    type Label = usize;
    type IntoIterator = crate::traits::labels::AssumeSortedIterator<
        std::iter::Copied<std::slice::Iter<'succ, Self::Label>>,
    >;
}

impl<'succ, D: Decode> Lending<'succ> for Iter<D> {
    type Lend = (usize, <Self as NodeLabelsLender<'succ>>::IntoIterator);
}

impl<D: Decode> Lender for Iter<D> {
    fn next(&mut self) -> Option<Lend<'_, Self>> {
        if self.current_node >= self.number_of_nodes {
            return None;
        }
        let mut res = self.backrefs.take(self.current_node);
        res.clear();
        self.get_successors_iter_priv(self.current_node, &mut res)
            .unwrap();

        let res = self.backrefs.replace(self.current_node, res);
        let node_id = self.current_node;
        self.current_node += 1;
        Some((node_id, unsafe {
            crate::traits::labels::AssumeSortedIterator::new(res.iter().copied())
        }))
    }
}

unsafe impl<D: Decode> SortedLender for Iter<D> {}

impl<D: Decode> ExactSizeLender for Iter<D> {
    fn len(&self) -> usize {
        self.number_of_nodes - self.current_node
    }
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Sebastiano Vigna
4	*
5	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6	*/
7
8	use std::path::PathBuf;
9
10	use super::*;
11	use crate::utils::CircularBuffer;
12	use anyhow::Result;
13	use bitflags::Flags;
14	use dsi_bitstream::codes::ToInt;
15	use dsi_bitstream::traits::BitSeek;
16	use dsi_bitstream::traits::BE;
17	use lender::*;
18
19	/// A sequential BvGraph that can be read from a `codes_reader_builder`.
20	/// The builder is needed because we should be able to create multiple iterators
21	/// and this allows us to have a single place where to store the mmapped file.
22	#[derive(Debug, Clone)]
23	pub struct BvGraphSeq<F> {
24	factory: F,
25	number_of_nodes: usize,
26	number_of_arcs: Option<u64>,
27	compression_window: usize,
28	min_interval_length: usize,
29	}
30
31	impl BvGraphSeq<()> {
32	pub fn with_basename(	31✔
33	basename: impl AsRef<std::path::Path>,
34	) -> LoadConfig<BE, Sequential, Dynamic, Mmap, Mmap> {
35	LoadConfig {
36	basename: PathBuf::from(basename.as_ref()),	124✔
37	graph_load_flags: Flags::empty(),	62✔
38	offsets_load_flags: Flags::empty(),	31✔
39	_marker: std::marker::PhantomData,
40	}
41	}
42	}
43
44	impl<F: SequentialDecoderFactory> SplitLabeling for BvGraphSeq<F>
45	where
46	for<'a> <F as SequentialDecoderFactory>::Decoder<'a>: Clone + Send + Sync,
47	{
48	type SplitLender<'a>
49	= split::seq::Lender<'a, BvGraphSeq<F>>
50	where
51	Self: 'a;
52	type IntoIterator<'a>
53	= split::seq::IntoIterator<'a, BvGraphSeq<F>>
54	where
55	Self: 'a;
56
57	fn split_iter(&self, how_many: usize) -> Self::IntoIterator<'_> {	10✔
58	split::seq::Iter::new(self.iter(), self.num_nodes(), how_many)	60✔
59	}
60	}
61
62	impl<F: SequentialDecoderFactory> SequentialLabeling for BvGraphSeq<F> {
63	type Label = usize;
64	type Lender<'a>
65	= Iter<F::Decoder<'a>>
66	where
67	Self: 'a;
68
69	#[inline(always)]
70	/// Returns the number of nodes in the graph
71	fn num_nodes(&self) -> usize {	20✔
72	self.number_of_nodes	20✔
73	}
74
75	#[inline(always)]
76	fn num_arcs_hint(&self) -> Option<u64> {	×
77	self.number_of_arcs	×
78	}
79
80	#[inline(always)]
81	fn iter_from(&self, from: usize) -> Self::Lender<'_> {	7,001✔
82	let mut iter = Iter::new(
83	self.factory.new_decoder().unwrap(),	21,003✔
84	self.number_of_nodes,	7,001✔
85	self.compression_window,	7,001✔
86	self.min_interval_length,	7,001✔
87	);
88
89	let _ = iter.advance_by(from);	14,002✔
90
91	iter	7,001✔
92	}
93	}
94
95	impl<F: SequentialDecoderFactory> SequentialGraph for BvGraphSeq<F> {}
96
97	impl<'a, F: SequentialDecoderFactory> IntoLender for &'a BvGraphSeq<F> {
98	type Lender = <BvGraphSeq<F> as SequentialLabeling>::Lender<'a>;
99
100	#[inline(always)]
101	fn into_lender(self) -> Self::Lender {	10✔
102	self.iter()	20✔
103	}
104	}
105
106	impl<F: SequentialDecoderFactory> BvGraphSeq<F> {
107	/// Creates a new sequential graph from a codes reader builder
108	/// and the number of nodes.
109	pub fn new(	6,983✔
110	codes_reader_builder: F,
111	number_of_nodes: usize,
112	number_of_arcs: Option<u64>,
113	compression_window: usize,
114	min_interval_length: usize,
115	) -> Self {
116	Self {
117	factory: codes_reader_builder,
118	number_of_nodes,
119	number_of_arcs,
120	compression_window,
121	min_interval_length,
122	}
123	}
124
125	#[inline(always)]
126	pub fn map_factory<F1, F0>(self, map_func: F0) -> BvGraphSeq<F1>	×
127	where
128	F0: FnOnce(F) -> F1,
129	F1: SequentialDecoderFactory,
130	{
131	BvGraphSeq {
132	factory: map_func(self.factory),	×
133	number_of_nodes: self.number_of_nodes,	×
134	number_of_arcs: self.number_of_arcs,	×
135	compression_window: self.compression_window,	×
136	min_interval_length: self.min_interval_length,	×
137	}
138	}
139
140	#[inline(always)]
141	/// Consume self and return the factory
142	pub fn into_inner(self) -> F {	×
143	self.factory	×
144	}
145	}
146
147	impl<F: SequentialDecoderFactory> BvGraphSeq<F>
148	where
149	for<'a> F::Decoder<'a>: Decode,
150	{
151	#[inline(always)]
152	/// Creates an iterator specialized in the degrees of the nodes.
153	/// This is slightly faster because it can avoid decoding some of the nodes
154	/// and completely skip the merging step.
155	pub fn offset_deg_iter(&self) -> OffsetDegIter<F::Decoder<'_>> {	6,961✔
156	OffsetDegIter::new(
157	self.factory.new_decoder().unwrap(),	20,883✔
158	self.number_of_nodes,	6,961✔
159	self.compression_window,	6,961✔
160	self.min_interval_length,	6,961✔
161	)
162	}
163	}
164
165	/// A fast sequential iterator over the nodes of the graph and their successors.
166	/// This iterator does not require to know the offsets of each node in the graph.
167	#[derive(Debug, Clone)]
168	pub struct Iter<D: Decode> {
169	pub(crate) number_of_nodes: usize,
170	pub(crate) compression_window: usize,
171	pub(crate) min_interval_length: usize,
172	pub(crate) decoder: D,
173	pub(crate) backrefs: CircularBuffer<Vec<usize>>,
174	pub(crate) current_node: usize,
175	}
176
177	impl<D: Decode + BitSeek> Iter<D> {
178	#[inline(always)]
179	/// Forward the call of `get_pos` to the inner `codes_reader`.
180	/// This returns the current bits offset in the bitstream.
181	pub fn bit_pos(&mut self) -> Result<u64, <D as BitSeek>::Error> {	1,175,880✔
182	self.decoder.bit_pos()	2,351,760✔
183	}
184	}
185
186	impl<D: Decode> Iter<D> {
187	/// Creates a new iterator from a codes reader
188	pub fn new(	38,214✔
189	decoder: D,
190	number_of_nodes: usize,
191	compression_window: usize,
192	min_interval_length: usize,
193	) -> Self {
194	Self {
195	number_of_nodes,
196	compression_window,
197	min_interval_length,
198	decoder,
199	backrefs: CircularBuffer::new(compression_window + 1),	38,214✔
200	current_node: 0,
201	}
202	}
203
204	/// Get the successors of the next node in the stream
205	pub fn next_successors(&mut self) -> Result<&[usize]> {	×
206	let mut res = self.backrefs.take(self.current_node);	×
207	res.clear();	×
208	self.get_successors_iter_priv(self.current_node, &mut res)?;	×
209	let res = self.backrefs.replace(self.current_node, res);	×
210	self.current_node += 1;	×
211	Ok(res)	×
212	}
213
214	#[inline(always)]
215	/// Inner method called by `next_successors` and the iterator `next` method
216	fn get_successors_iter_priv(&mut self, node_id: usize, results: &mut Vec<usize>) -> Result<()> {	94,686,926✔
217	let degree = self.decoder.read_outdegree() as usize;	189,373,852✔
218	// no edges, we are done!
219	if degree == 0 {	94,686,926✔
220	return Ok(());	18,914,596✔
221	}
222
223	// ensure that we have enough capacity in the vector for not reallocating
224	results.reserve(degree.saturating_sub(results.capacity()));	×
225	// read the reference offset
226	let ref_delta = if self.compression_window != 0 {	×
227	self.decoder.read_reference_offset() as usize	74,251,931✔
228	} else {
229	0	1,520,399✔
230	};
231	// if we copy nodes from a previous one
232	if ref_delta != 0 {	×
233	// compute the node id of the reference
234	let reference_node_id = node_id - ref_delta;	106,280,246✔
235	// retrieve the data
236	let neighbours = &self.backrefs[reference_node_id];	106,280,246✔
237	//debug_assert!(!neighbours.is_empty());
238	// get the info on which destinations to copy
239	let number_of_blocks = self.decoder.read_block_count() as usize;	106,280,246✔
240	// no blocks, we copy everything
241	if number_of_blocks == 0 {	72,489,770✔
242	results.extend_from_slice(neighbours);	38,699,294✔
243	} else {
244	// otherwise we copy only the blocks of even index
245	// the first block could be zero
246	let mut idx = self.decoder.read_block() as usize;	33,790,476✔
247	results.extend_from_slice(&neighbours[..idx]);	×
248
249	// while the other can't
250	for block_id in 1..number_of_blocks {	50,542,246✔
251	let block = self.decoder.read_block() as usize;	×
252	let end = idx + block + 1;	×
253	if block_id % 2 == 0 {	15,257,439✔
254	results.extend_from_slice(&neighbours[idx..end]);	61,029,756✔
255	}
256	idx = end;	×
257	}
258	if number_of_blocks & 1 == 0 {	20,027,368✔
259	results.extend_from_slice(&neighbours[idx..]);	60,082,104✔
260	}
261	}
262	};
263
264	// if we still have to read nodes
265	let nodes_left_to_decode = degree - results.len();	×
266	if nodes_left_to_decode != 0 && self.min_interval_length != 0 {	57,704,494✔
267	// read the number of intervals
268	let number_of_intervals = self.decoder.read_interval_count() as usize;	112,034,472✔
269	if number_of_intervals != 0 {	56,017,236✔
270	// pre-allocate with capacity for efficiency
271	let node_id_offset = self.decoder.read_interval_start().to_int();	36,267,152✔
272	let mut start = (node_id as i64 + node_id_offset) as usize;	18,133,576✔
273	let mut delta = self.decoder.read_interval_len() as usize;	18,133,576✔
274	delta += self.min_interval_length;	9,066,788✔
275	// save the first interval
276	results.extend(start..(start + delta));	36,267,152✔
277	start += delta;	9,066,788✔
278	// decode the intervals
279	for _ in 1..number_of_intervals {	14,087,404✔
280	start += 1 + self.decoder.read_interval_start() as usize;	5,020,616✔
281	delta = self.decoder.read_interval_len() as usize;	5,020,616✔
282	delta += self.min_interval_length;	5,020,616✔
283
284	results.extend(start..(start + delta));	5,020,616✔
285
286	start += delta;	5,020,616✔
287	}
288	}
289	}
290
291	// decode the extra nodes if needed
292	let nodes_left_to_decode = degree - results.len();	×
293	if nodes_left_to_decode != 0 {	×
294	// pre-allocate with capacity for efficiency
295	let node_id_offset = self.decoder.read_first_residual().to_int();	225,670,692✔
296	let mut extra = (node_id as i64 + node_id_offset) as usize;	112,835,346✔
297	results.push(extra);	169,253,019✔
298	// decode the successive extra nodes
299	for _ in 1..nodes_left_to_decode {	201,228,796✔
300	extra += 1 + self.decoder.read_residual() as usize;	144,811,123✔
301	results.push(extra);	144,811,123✔
302	}
303	}
304
305	results.sort();	×
306	Ok(())	×
307	}
308	}
309
310	impl<'succ, D: Decode> NodeLabelsLender<'succ> for Iter<D> {
311	type Label = usize;
312	type IntoIterator = crate::traits::labels::AssumeSortedIterator<
313	std::iter::Copied<std::slice::Iter<'succ, Self::Label>>,
314	>;
315	}
316
317	impl<'succ, D: Decode> Lending<'succ> for Iter<D> {
318	type Lend = (usize, <Self as NodeLabelsLender<'succ>>::IntoIterator);
319	}
320
321	impl<D: Decode> Lender for Iter<D> {
322	fn next(&mut self) -> Option<Lend<'_, Self>> {	94,687,058✔
323	if self.current_node >= self.number_of_nodes {	94,687,058✔
324	return None;	132✔
325	}
326	let mut res = self.backrefs.take(self.current_node);	×
327	res.clear();	×
328	self.get_successors_iter_priv(self.current_node, &mut res)	×
329	.unwrap();
330
331	let res = self.backrefs.replace(self.current_node, res);	×
332	let node_id = self.current_node;	×
333	self.current_node += 1;	×
334	Some((node_id, unsafe {	×
335	crate::traits::labels::AssumeSortedIterator::new(res.iter().copied())	×
336	}))
337	}
338	}
339
340	unsafe impl<D: Decode> SortedLender for Iter<D> {}
341
342	impl<D: Decode> ExactSizeLender for Iter<D> {
343	fn len(&self) -> usize {	×
344	self.number_of_nodes - self.current_node	×
345	}
346	}

vigna / webgraph-rs / 15788522925

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous