18761474399

Committed 23 Oct 2025 08:43PM UTC coverage: 61.976% (-0.06%) from 62.035%

Build # 18761474399

Build Type

push

github

Committed by

vigna

Commit Message

Goodbye le_bins/be_bins

Coverage Stats

5201 of 8392 relevant lines covered (61.98%)

27888380.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.76

/cli/src/to/bvgraph.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use crate::create_parent_dir;
use crate::*;
use anyhow::Result;
use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
use dsi_bitstream::prelude::*;

use mmap_rs::MmapFlags;
use std::path::PathBuf;
use tempfile::Builder;
use webgraph::prelude::*;

#[derive(Parser, Debug)]
#[command(name = "bvgraph", about = "Recompresses a BvGraph, possibly applying a permutation to its node identifiers.", long_about = None)]
pub struct CliArgs {
    /// The basename of the source graph.
    pub src: PathBuf,
    /// The basename of the destination graph.
    pub dst: PathBuf,

    #[clap(flatten)]
    pub num_threads: NumThreadsArg,

    #[arg(long)]
    /// The path to an optional permutation in binary big-endian format to be applied to the graph.
    pub permutation: Option<PathBuf>,

    #[clap(flatten)]
    pub memory_usage: MemoryUsageArg,

    #[clap(flatten)]
    pub ca: CompressArgs,
}

pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
    create_parent_dir(&args.dst)?;

    let permutation = if let Some(path) = args.permutation.as_ref() {
        Some(JavaPermutation::mmap(path, MmapFlags::RANDOM_ACCESS)?)
    } else {
        None
    };

    let target_endianness = args.ca.endianness.clone();
    match get_endianness(&args.src)?.as_str() {
        BE::NAME => compress::<BE>(global_args, args, target_endianness, permutation)?,
        LE::NAME => compress::<LE>(global_args, args, target_endianness, permutation)?,
        e => panic!("Unknown endianness: {}", e),
    };
    Ok(())
}

pub fn compress<E: Endianness>(
    _global_args: GlobalArgs,
    args: CliArgs,
    target_endianness: Option<String>,
    permutation: Option<JavaPermutation>,
) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Send + Sync + Clone,
{
    let dir = Builder::new().prefix("to_bvgraph_").tempdir()?;

    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);

    if args.src.with_extension(EF_EXTENSION).exists() {
        let graph = BvGraph::with_basename(&args.src).endianness::<E>().load()?;

        if let Some(permutation) = permutation {
            let memory_usage = args.memory_usage.memory_usage;

            log::info!("Permuting graph with memory usage {}", memory_usage);
            let start = std::time::Instant::now();
            let sorted = webgraph::transform::permute_split(
                &graph,
                &permutation,
                memory_usage,
                &thread_pool,
            )?;
            log::info!(
                "Permuted the graph. It took {:.3} seconds",
                start.elapsed().as_secs_f64()
            );
            BvComp::parallel_endianness(
                args.dst,
                &sorted,
                sorted.num_nodes(),
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness.unwrap_or_else(|| E::NAME.into()),
            )?;
        } else {
            BvComp::parallel_endianness(
                args.dst,
                &graph,
                graph.num_nodes(),
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness.unwrap_or_else(|| E::NAME.into()),
            )?;
        }
    } else {
        log::warn!(
            "The .ef file does not exist. The graph will be sequentially which will result in slower compression. If you can, run `build_ef` before recompressing."
        );
        let seq_graph = BvGraphSeq::with_basename(&args.src)
            .endianness::<E>()
            .load()?;

        if let Some(permutation) = permutation {
            let memory_usage = args.memory_usage.memory_usage;

            log::info!("Permuting graph with memory usage {}", memory_usage);
            let start = std::time::Instant::now();
            let permuted = webgraph::transform::permute(&seq_graph, &permutation, memory_usage)?;
            log::info!(
                "Permuted the graph. It took {:.3} seconds",
                start.elapsed().as_secs_f64()
            );

            BvComp::parallel_endianness(
                args.dst,
                &permuted,
                permuted.num_nodes(),
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness.unwrap_or_else(|| E::NAME.into()),
            )?;
        } else {
            BvComp::parallel_endianness(
                args.dst,
                &seq_graph,
                seq_graph.num_nodes(),
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness.unwrap_or_else(|| E::NAME.into()),
            )?;
        }
    }
    Ok(())
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Tommaso Fontana
4	*
5	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6	*/
7
8	use crate::create_parent_dir;
9	use crate::*;
10	use anyhow::Result;
11	use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
12	use dsi_bitstream::prelude::*;
13
14	use mmap_rs::MmapFlags;
15	use std::path::PathBuf;
16	use tempfile::Builder;
17	use webgraph::prelude::*;
18
19	#[derive(Parser, Debug)]
20	#[command(name = "bvgraph", about = "Recompresses a BvGraph, possibly applying a permutation to its node identifiers.", long_about = None)]
21	pub struct CliArgs {
22	/// The basename of the source graph.
23	pub src: PathBuf,
24	/// The basename of the destination graph.
25	pub dst: PathBuf,
26
27	#[clap(flatten)]
28	pub num_threads: NumThreadsArg,
29
30	#[arg(long)]
31	/// The path to an optional permutation in binary big-endian format to be applied to the graph.
32	pub permutation: Option<PathBuf>,
33
34	#[clap(flatten)]
35	pub memory_usage: MemoryUsageArg,
36
37	#[clap(flatten)]
38	pub ca: CompressArgs,
39	}
40
41	pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {	4✔
42	create_parent_dir(&args.dst)?;	8✔
43
44	let permutation = if let Some(path) = args.permutation.as_ref() {	12✔
45	Some(JavaPermutation::mmap(path, MmapFlags::RANDOM_ACCESS)?)	8✔
46	} else {
47	None	×
48	};
49
50	let target_endianness = args.ca.endianness.clone();	12✔
51	match get_endianness(&args.src)?.as_str() {	12✔
52	BE::NAME => compress::<BE>(global_args, args, target_endianness, permutation)?,	24✔
53	LE::NAME => compress::<LE>(global_args, args, target_endianness, permutation)?,	×
54	e => panic!("Unknown endianness: {}", e),	×
55	};
56	Ok(())	4✔
57	}
58
59	pub fn compress<E: Endianness>(	4✔
60	_global_args: GlobalArgs,
61	args: CliArgs,
62	target_endianness: Option<String>,
63	permutation: Option<JavaPermutation>,
64	) -> Result<()>
65	where
66	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
67	for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Send + Sync + Clone,
68	{
69	let dir = Builder::new().prefix("to_bvgraph_").tempdir()?;	16✔
70
71	let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);	12✔
72
73	if args.src.with_extension(EF_EXTENSION).exists() {	4✔
74	let graph = BvGraph::with_basename(&args.src).endianness::<E>().load()?;	20✔
75
76	if let Some(permutation) = permutation {	8✔
77	let memory_usage = args.memory_usage.memory_usage;	8✔
78
79	log::info!("Permuting graph with memory usage {}", memory_usage);	8✔
80	let start = std::time::Instant::now();	8✔
81	let sorted = webgraph::transform::permute_split(
82	&graph,	4✔
83	&permutation,	4✔
84	memory_usage,	4✔
85	&thread_pool,	4✔
86	)?;
87	log::info!(	4✔
88	"Permuted the graph. It took {:.3} seconds",	4✔
89	start.elapsed().as_secs_f64()	8✔
90	);
91	BvComp::parallel_endianness(
92	args.dst,	4✔
93	&sorted,	4✔
94	sorted.num_nodes(),	8✔
95	args.ca.into(),	8✔
96	&thread_pool,	4✔
97	dir,	4✔
98	&target_endianness.unwrap_or_else(\|\| E::NAME.into()),	16✔
99	)?;
100	} else {
101	BvComp::parallel_endianness(
102	args.dst,	×
103	&graph,	×
104	graph.num_nodes(),	×
105	args.ca.into(),	×
106	&thread_pool,	×
107	dir,	×
108	&target_endianness.unwrap_or_else(\|\| E::NAME.into()),	×
109	)?;
110	}
111	} else {
112	log::warn!(	×
113	"The .ef file does not exist. The graph will be sequentially which will result in slower compression. If you can, run `build_ef` before recompressing."	×
114	);
115	let seq_graph = BvGraphSeq::with_basename(&args.src)	×
116	.endianness::<E>()
117	.load()?;
118
119	if let Some(permutation) = permutation {	×
120	let memory_usage = args.memory_usage.memory_usage;	×
121
122	log::info!("Permuting graph with memory usage {}", memory_usage);	×
123	let start = std::time::Instant::now();	×
124	let permuted = webgraph::transform::permute(&seq_graph, &permutation, memory_usage)?;	×
125	log::info!(	×
126	"Permuted the graph. It took {:.3} seconds",	×
127	start.elapsed().as_secs_f64()	×
128	);
129
130	BvComp::parallel_endianness(
131	args.dst,	×
132	&permuted,	×
133	permuted.num_nodes(),	×
134	args.ca.into(),	×
135	&thread_pool,	×
136	dir,	×
137	&target_endianness.unwrap_or_else(\|\| E::NAME.into()),	×
138	)?;
139	} else {
140	BvComp::parallel_endianness(
141	args.dst,	×
142	&seq_graph,	×
143	seq_graph.num_nodes(),	×
144	args.ca.into(),	×
145	&thread_pool,	×
146	dir,	×
147	&target_endianness.unwrap_or_else(\|\| E::NAME.into()),	×
148	)?;
149	}
150	}
151	Ok(())	4✔
152	}

vigna / webgraph-rs / 18761474399

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous