23365769388

Committed 20 Mar 2026 10:52PM UTC coverage: 68.228% (-3.0%) from 71.245%

Build # 23365769388

Build Type

push

github

Committed by

vigna

Commit Message

No le_bins,be_bins for webgraph

Coverage Stats

6655 of 9754 relevant lines covered (68.23%)

46582760.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/cli/src/check/ef.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
 * SPDX-FileCopyrightText: 2026 Sebastiano Vigna
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use crate::LogIntervalArg;
use anyhow::{Context, Result};
use clap::Parser;
use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
use dsi_bitstream::prelude::*;
use dsi_progress_logger::prelude::*;
use epserde::prelude::*;
use log::info;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use sux::traits::IndexedSeq;
use webgraph::graphs::bvgraph::get_endianness;
use webgraph::graphs::bvgraph::{EF, EF_EXTENSION, OFFSETS_EXTENSION, PROPERTIES_EXTENSION};
use webgraph::prelude::*;

#[derive(Parser, Debug)]
#[command(name = "ef", about = "Checks that the \".ef\" file (and \".offsets\" if present) is consistent with the graph.", long_about = None, next_line_help = true)]
pub struct CliArgs {
    /// The basename of the graph.​
    pub basename: PathBuf,

    #[clap(flatten)]
    pub log_interval: LogIntervalArg,
}

pub fn main(args: CliArgs) -> Result<()> {
    match get_endianness(&args.basename)?.as_str() {
        #[cfg(feature = "be_bins")]
        BE::NAME => check_ef::<BE>(args),
        #[cfg(feature = "le_bins")]
        LE::NAME => check_ef::<LE>(args),
        e => panic!("Unknown endianness: {}", e),
    }
}

pub fn check_ef<E: Endianness + 'static>(args: CliArgs) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek,
{
    let properties_path = args.basename.with_extension(PROPERTIES_EXTENSION);
    let f = File::open(&properties_path).with_context(|| {
        format!(
            "Could not load properties file: {}",
            properties_path.display()
        )
    })?;
    let map = java_properties::read(BufReader::new(f))?;
    let num_nodes = map.get("nodes").unwrap().parse::<usize>()?;

    // Creates the offsets file
    let of_file_path = args.basename.with_extension(OFFSETS_EXTENSION);

    let ef = unsafe { EF::mmap(args.basename.with_extension(EF_EXTENSION), Flags::default()) }?;
    let ef = ef.uncase();

    let mut pl = progress_logger![
        display_memory = true,
        item_name = "offset",
        expected_updates = Some(num_nodes),
    ];
    if let Some(duration) = args.log_interval.log_interval {
        pl.log_interval(duration);
    }

    // if the offset files exists, read it to build elias-fano
    if of_file_path.exists() {
        // create a bit reader on the file
        let mut reader = buf_bit_reader::from_path::<BE, u32>(of_file_path)?;
        // progress bar
        pl.start("Checking offsets file against Elias–Fano...");
        // read the graph a write the offsets
        let mut offset = 0;
        for node_id in 0..num_nodes + 1 {
            // write where
            offset += reader.read_gamma()?;
            // read ef
            let ef_res = ef.get(node_id as _);
            assert_eq!(offset, ef_res, "node_id: {}", node_id);
            // decode the next nodes so we know where the next node_id starts
            pl.light_update();
        }
    } else {
        info!("No offsets file, checking against graph file only");
    }

    let mut pl = progress_logger![
        display_memory = true,
        item_name = "offset",
        expected_updates = Some(num_nodes),
    ];
    if let Some(duration) = args.log_interval.log_interval {
        pl.log_interval(duration);
    }

    let seq_graph =
        webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.basename)
            .endianness::<E>()
            .load()?;
    // otherwise directly read the graph
    // progress bar
    pl.start("Checking graph against Elias–Fano...");
    // read the graph a write the offsets
    for (node, (new_offset, _degree)) in seq_graph.offset_deg_iter().enumerate() {
        // decode the next nodes so we know where the next node_id starts
        // read ef
        let ef_res = ef.get(node as _);
        assert_eq!(new_offset, ef_res, "node_id: {}", node);
        pl.light_update();
    }
    pl.done();
    Ok(())
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Tommaso Fontana
4	* SPDX-FileCopyrightText: 2026 Sebastiano Vigna
5	*
6	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
7	*/
8
9	use crate::LogIntervalArg;
10	use anyhow::{Context, Result};
11	use clap::Parser;
12	use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
13	use dsi_bitstream::prelude::*;
14	use dsi_progress_logger::prelude::*;
15	use epserde::prelude::*;
16	use log::info;
17	use std::fs::File;
18	use std::io::BufReader;
19	use std::path::PathBuf;
20	use sux::traits::IndexedSeq;
21	use webgraph::graphs::bvgraph::get_endianness;
22	use webgraph::graphs::bvgraph::{EF, EF_EXTENSION, OFFSETS_EXTENSION, PROPERTIES_EXTENSION};
23	use webgraph::prelude::*;
24
25	#[derive(Parser, Debug)]
26	#[command(name = "ef", about = "Checks that the \".ef\" file (and \".offsets\" if present) is consistent with the graph.", long_about = None, next_line_help = true)]
27	pub struct CliArgs {
28	/// The basename of the graph.
29	pub basename: PathBuf,
30
31	#[clap(flatten)]
32	pub log_interval: LogIntervalArg,
33	}
34
35	pub fn main(args: CliArgs) -> Result<()> {	×
36	match get_endianness(&args.basename)?.as_str() {	×
37	#[cfg(feature = "be_bins")]
38	BE::NAME => check_ef::<BE>(args),	×
39	#[cfg(feature = "le_bins")]
40	LE::NAME => check_ef::<LE>(args),	×
41	e => panic!("Unknown endianness: {}", e),	×
42	}
43	}
44
45	pub fn check_ef<E: Endianness + 'static>(args: CliArgs) -> Result<()>	×
46	where
47	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
48	for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek,
49	{
50	let properties_path = args.basename.with_extension(PROPERTIES_EXTENSION);	×
51	let f = File::open(&properties_path).with_context(\|\| {	×
52	format!(	×
53	"Could not load properties file: {}",	×
54	properties_path.display()	×
55	)
56	})?;
57	let map = java_properties::read(BufReader::new(f))?;	×
58	let num_nodes = map.get("nodes").unwrap().parse::<usize>()?;	×
59
60	// Creates the offsets file
61	let of_file_path = args.basename.with_extension(OFFSETS_EXTENSION);	×
62
63	let ef = unsafe { EF::mmap(args.basename.with_extension(EF_EXTENSION), Flags::default()) }?;	×
64	let ef = ef.uncase();	×
65
66	let mut pl = progress_logger![	×
67	display_memory = true,	×
68	item_name = "offset",	×
69	expected_updates = Some(num_nodes),	×
70	];
71	if let Some(duration) = args.log_interval.log_interval {	×
72	pl.log_interval(duration);	×
73	}
74
75	// if the offset files exists, read it to build elias-fano
76	if of_file_path.exists() {	×
77	// create a bit reader on the file
78	let mut reader = buf_bit_reader::from_path::<BE, u32>(of_file_path)?;	×
79	// progress bar
80	pl.start("Checking offsets file against Elias–Fano...");	×
81	// read the graph a write the offsets
82	let mut offset = 0;	×
83	for node_id in 0..num_nodes + 1 {	×
84	// write where
85	offset += reader.read_gamma()?;	×
86	// read ef
87	let ef_res = ef.get(node_id as _);	×
88	assert_eq!(offset, ef_res, "node_id: {}", node_id);	×
89	// decode the next nodes so we know where the next node_id starts
90	pl.light_update();	×
91	}
92	} else {
93	info!("No offsets file, checking against graph file only");	×
94	}
95
96	let mut pl = progress_logger![	×
97	display_memory = true,	×
98	item_name = "offset",	×
99	expected_updates = Some(num_nodes),	×
100	];
101	if let Some(duration) = args.log_interval.log_interval {	×
102	pl.log_interval(duration);	×
103	}
104
105	let seq_graph =	×
106	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.basename)	×
107	.endianness::<E>()
108	.load()?;
109	// otherwise directly read the graph
110	// progress bar
111	pl.start("Checking graph against Elias–Fano...");	×
112	// read the graph a write the offsets
113	for (node, (new_offset, _degree)) in seq_graph.offset_deg_iter().enumerate() {	×
114	// decode the next nodes so we know where the next node_id starts
115	// read ef
116	let ef_res = ef.get(node as _);	×
117	assert_eq!(new_offset, ef_res, "node_id: {}", node);	×
118	pl.light_update();	×
119	}
120	pl.done();	×
121	Ok(())	×
122	}

vigna / webgraph-rs / 23365769388

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous