22349485807

Committed 24 Feb 2026 11:47AM UTC coverage: 71.75% (+0.09%) from 71.661%

Build # 22349485807

Build Type

push

github

Committed by

vigna

Commit Message

Remove useless arguments/fields, and replaced some erroneous num_cpus::get with rayon::current_num_threads

Run Details

8 of 18 new or added lines in 7 files covered. (44.44%)

1 existing line in 1 file now uncovered.

6286 of 8761 relevant lines covered (71.75%)

52344897.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.78

/cli/src/transform/simplify.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use crate::*;
use anyhow::Result;
use dsi_bitstream::{dispatch::factory::CodesReaderFactoryHelper, prelude::*};
use mmap_rs::MmapFlags;
use std::path::PathBuf;
use tempfile::Builder;
use webgraph::graphs::union_graph::UnionGraph;
use webgraph::prelude::*;

#[derive(Parser, Debug)]
#[command(name = "simplify", about = "Makes a BvGraph simple (undirected and loopless) by adding missing arcs and removing loops, optionally applying a permutation.", long_about = None)]
pub struct CliArgs {
    /// The basename of the graph.
    pub src: PathBuf,
    /// The basename of the simplified graph.
    pub dst: PathBuf,

    #[arg(long)]
    /// The basename of a pre-computed transposed version of the source graph, which
    /// will be use to speed up the simplification.
    pub transposed: Option<PathBuf>,

    #[clap(flatten)]
    pub num_threads: NumThreadsArg,

    #[clap(flatten)]
    pub memory_usage: MemoryUsageArg,

    #[clap(flatten)]
    pub ca: CompressArgs,

    #[arg(long)]
    /// The path to an optional permutation in binary big-endian format to apply to the graph.
    pub permutation: Option<PathBuf>,
}

pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
    create_parent_dir(&args.dst)?;

    match get_endianness(&args.src)?.as_str() {
        #[cfg(feature = "be_bins")]
        BE::NAME => simplify::<BE>(global_args, args),
        #[cfg(feature = "le_bins")]
        LE::NAME => simplify::<LE>(global_args, args),
        e => panic!("Unknown endianness: {}", e),
    }
}

fn no_ef_warn(basepath: impl AsRef<std::path::Path>) {
    log::warn!(SEQ_PROC_WARN![], basepath.as_ref().display());
}

pub fn simplify<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Clone + Send + Sync,
{
    // TODO!: speed it up by using random access graph if possible
    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);

    let target_endianness = args.ca.endianness.clone().unwrap_or_else(|| E::NAME.into());

    let dir = Builder::new().prefix("transform_simplify_").tempdir()?;
    let chunk_size = args.ca.chunk_size;
    let bvgraphz = args.ca.bvgraphz;
    let mut builder = BvCompConfig::new(&args.dst)
        .with_comp_flags(args.ca.into())
        .with_tmp_dir(&dir);

    if bvgraphz {
        builder = builder.with_chunk_size(chunk_size);
    }

    match (args.permutation, args.transposed) {
        // load the transposed graph and use it to directly compress the graph
        // without doing any sorting
        (None, Some(t_path)) => {
            log::info!("Transposed graph provided, using it to simplify the graph");

            let has_ef_graph =
                std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file());
            let has_ef_t_graph =
                std::fs::metadata(t_path.with_extension("ef")).is_ok_and(|x| x.is_file());

            match (has_ef_graph, has_ef_t_graph) {
                (true, true) => {
                    log::info!("Both .ef files found, using simplify split");

                    let graph =
                        webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                            .endianness::<E>()
                            .load()?;
                    let num_nodes = graph.num_nodes();
                    let graph_t =
                        webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&t_path)
                            .endianness::<E>()
                            .load()?;

                    if graph_t.num_nodes() != num_nodes {
                        anyhow::bail!(
                            "The number of nodes in the graph and its transpose do not match! {} != {}",
                            num_nodes,
                            graph_t.num_nodes()
                        );
                    }

                    let sorted = NoSelfLoopsGraph(UnionGraph(graph, graph_t));

                    thread_pool.install(|| {
                        builder.par_comp_lenders_endianness(&sorted, &target_endianness)
                    })?;

                    return Ok(());
                }
                (true, false) => {
                    no_ef_warn(&t_path);
                }
                (false, true) => {
                    no_ef_warn(&args.src);
                }
                (false, false) => {
                    no_ef_warn(&args.src);
                    no_ef_warn(&t_path);
                }
            }

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;
            let num_nodes = seq_graph.num_nodes();
            let seq_graph_t =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&t_path)
                    .endianness::<E>()
                    .load()?;

            if seq_graph_t.num_nodes() != num_nodes {
                anyhow::bail!(
                    "The number of nodes in the graph and its transpose do not match! {} != {}",
                    num_nodes,
                    seq_graph_t.num_nodes()
                );
            }

            let sorted = NoSelfLoopsGraph(UnionGraph(seq_graph, seq_graph_t));

            thread_pool
                .install(|| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;
        }
        // apply the permutation, don't care if the transposed graph is already computed
        // as we cannot really exploit it
        (Some(perm_path), None | Some(_)) => {
            log::info!("Permutation provided, applying it to the graph");

            let perm = JavaPermutation::mmap(perm_path, MmapFlags::RANDOM_ACCESS)?;

            // if the .ef file exists, we can use the simplify split
            if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file()) {
                log::info!(".ef file found, using simplify split");
                let graph =
                    webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                        .endianness::<E>()
                        .load()?;

                let perm_graph = PermutedGraph {
                    graph: &graph,
                    perm: &perm,
                };

                thread_pool.install(|| {
                    let sorted = webgraph::transform::simplify_split(
                        &perm_graph,
                        args.memory_usage.memory_usage,
                    )?;

                    builder.par_comp_lenders_endianness(&sorted, &target_endianness)
                })?;

                return Ok(());
            }

            no_ef_warn(&args.src);

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;

            let perm_graph = PermutedGraph {
                graph: &seq_graph,
                perm: &perm,
            };

            // simplify the graph
            let sorted =
                webgraph::transform::simplify(&perm_graph, args.memory_usage.memory_usage)?;

            thread_pool
                .install(|| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;
        }
        // just compute the transpose on the fly
        (None, None) => {
            log::info!(
                "No permutation or transposed graph provided, computing the transpose on the fly"
            );
            // if the .ef file exists, we can use the simplify split
            if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file()) {
                log::info!(".ef file found, using simplify split");

                let graph =
                    webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                        .endianness::<E>()
                        .load()?;

                thread_pool.install(|| {
                    let sorted = webgraph::transform::simplify_split(
                        &graph,
                        args.memory_usage.memory_usage,
                    )?;

                    builder.par_comp_lenders_endianness(&sorted, &target_endianness)
                })?;

                return Ok(());
            }

            no_ef_warn(&args.src);

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;

            // transpose the graph
            let sorted =
                webgraph::transform::simplify_sorted(seq_graph, args.memory_usage.memory_usage)?;

            thread_pool
                .install(|| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;
        }
    }

    Ok(())
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Tommaso Fontana
4	*
5	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6	*/
7
8	use crate::*;
9	use anyhow::Result;
10	use dsi_bitstream::{dispatch::factory::CodesReaderFactoryHelper, prelude::*};
11	use mmap_rs::MmapFlags;
12	use std::path::PathBuf;
13	use tempfile::Builder;
14	use webgraph::graphs::union_graph::UnionGraph;
15	use webgraph::prelude::*;
16
17	#[derive(Parser, Debug)]
18	#[command(name = "simplify", about = "Makes a BvGraph simple (undirected and loopless) by adding missing arcs and removing loops, optionally applying a permutation.", long_about = None)]
19	pub struct CliArgs {
20	/// The basename of the graph.
21	pub src: PathBuf,
22	/// The basename of the simplified graph.
23	pub dst: PathBuf,
24
25	#[arg(long)]
26	/// The basename of a pre-computed transposed version of the source graph, which
27	/// will be use to speed up the simplification.
28	pub transposed: Option<PathBuf>,
29
30	#[clap(flatten)]
31	pub num_threads: NumThreadsArg,
32
33	#[clap(flatten)]
34	pub memory_usage: MemoryUsageArg,
35
36	#[clap(flatten)]
37	pub ca: CompressArgs,
38
39	#[arg(long)]
40	/// The path to an optional permutation in binary big-endian format to apply to the graph.
41	pub permutation: Option<PathBuf>,
42	}
43
44	pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {	5✔
45	create_parent_dir(&args.dst)?;	10✔
46
47	match get_endianness(&args.src)?.as_str() {	15✔
48	#[cfg(feature = "be_bins")]
49	BE::NAME => simplify::<BE>(global_args, args),	20✔
50	#[cfg(feature = "le_bins")]
51	LE::NAME => simplify::<LE>(global_args, args),	×
52	e => panic!("Unknown endianness: {}", e),	×
53	}
54	}
55
56	fn no_ef_warn(basepath: impl AsRef<std::path::Path>) {	×
57	log::warn!(SEQ_PROC_WARN![], basepath.as_ref().display());	×
58	}
59
60	pub fn simplify<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>	5✔
61	where
62	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
63	for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Clone + Send + Sync,
64	{
65	// TODO!: speed it up by using random access graph if possible
66	let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);	15✔
67
68	let target_endianness = args.ca.endianness.clone().unwrap_or_else(\|\| E::NAME.into());	30✔
69
70	let dir = Builder::new().prefix("transform_simplify_").tempdir()?;	20✔
71	let chunk_size = args.ca.chunk_size;	10✔
72	let bvgraphz = args.ca.bvgraphz;	10✔
73	let mut builder = BvCompConfig::new(&args.dst)	15✔
74	.with_comp_flags(args.ca.into())	15✔
75	.with_tmp_dir(&dir);	10✔
76
77	if bvgraphz {	5✔
78	builder = builder.with_chunk_size(chunk_size);	×
79	}
80
81	match (args.permutation, args.transposed) {	10✔
82	// load the transposed graph and use it to directly compress the graph
83	// without doing any sorting
84	(None, Some(t_path)) => {	×
85	log::info!("Transposed graph provided, using it to simplify the graph");	×
86
87	let has_ef_graph =	×
88	std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file());	×
89	let has_ef_t_graph =	×
90	std::fs::metadata(t_path.with_extension("ef")).is_ok_and(\|x\| x.is_file());	×
91
92	match (has_ef_graph, has_ef_t_graph) {	×
93	(true, true) => {	×
94	log::info!("Both .ef files found, using simplify split");	×
95
96	let graph =	×
97	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	×
98	.endianness::<E>()
99	.load()?;
100	let num_nodes = graph.num_nodes();	×
101	let graph_t =	×
102	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&t_path)	×
103	.endianness::<E>()
104	.load()?;
105
106	if graph_t.num_nodes() != num_nodes {	×
107	anyhow::bail!(	×
108	"The number of nodes in the graph and its transpose do not match! {} != {}",	×
109	num_nodes,	×
110	graph_t.num_nodes()	×
111	);
112	}
113
114	let sorted = NoSelfLoopsGraph(UnionGraph(graph, graph_t));	×
115
116	thread_pool.install(\|\| {	×
NEW 117	builder.par_comp_lenders_endianness(&sorted, &target_endianness)	×
118	})?;
119
120	return Ok(());	×
121	}
122	(true, false) => {	×
123	no_ef_warn(&t_path);	×
124	}
125	(false, true) => {	×
126	no_ef_warn(&args.src);	×
127	}
128	(false, false) => {	×
129	no_ef_warn(&args.src);	×
130	no_ef_warn(&t_path);	×
131	}
132	}
133
134	let seq_graph =	×
135	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
136	.endianness::<E>()
137	.load()?;
138	let num_nodes = seq_graph.num_nodes();	×
139	let seq_graph_t =	×
140	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&t_path)	×
141	.endianness::<E>()
142	.load()?;
143
144	if seq_graph_t.num_nodes() != num_nodes {	×
145	anyhow::bail!(	×
146	"The number of nodes in the graph and its transpose do not match! {} != {}",	×
147	num_nodes,	×
148	seq_graph_t.num_nodes()	×
149	);
150	}
151
152	let sorted = NoSelfLoopsGraph(UnionGraph(seq_graph, seq_graph_t));	×
153
NEW 154	thread_pool	×
NEW 155	.install(\|\| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;	×
156	}
157	// apply the permutation, don't care if the transposed graph is already computed
158	// as we cannot really exploit it
159	(Some(perm_path), None \| Some(_)) => {	5✔
160	log::info!("Permutation provided, applying it to the graph");	5✔
161
162	let perm = JavaPermutation::mmap(perm_path, MmapFlags::RANDOM_ACCESS)?;	15✔
163
164	// if the .ef file exists, we can use the simplify split
165	if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file()) {	25✔
166	log::info!(".ef file found, using simplify split");	5✔
167	let graph =	5✔
168	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	10✔
169	.endianness::<E>()
170	.load()?;
171
172	let perm_graph = PermutedGraph {
173	graph: &graph,	5✔
174	perm: &perm,	5✔
175	};
176
177	thread_pool.install(\|\| {	15✔
178	let sorted = webgraph::transform::simplify_split(	10✔
179	&perm_graph,	5✔
180	args.memory_usage.memory_usage,	5✔
181	)?;
182
183	builder.par_comp_lenders_endianness(&sorted, &target_endianness)	20✔
184	})?;
185
186	return Ok(());	5✔
187	}
188
189	no_ef_warn(&args.src);	×
190
191	let seq_graph =	×
192	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
193	.endianness::<E>()
194	.load()?;
195
196	let perm_graph = PermutedGraph {
197	graph: &seq_graph,	×
198	perm: &perm,	×
199	};
200
201	// simplify the graph
202	let sorted =	×
203	webgraph::transform::simplify(&perm_graph, args.memory_usage.memory_usage)?;	×
204
NEW 205	thread_pool	×
NEW 206	.install(\|\| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;	×
207	}
208	// just compute the transpose on the fly
209	(None, None) => {	×
210	log::info!(	×
211	"No permutation or transposed graph provided, computing the transpose on the fly"	×
212	);
213	// if the .ef file exists, we can use the simplify split
214	if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file()) {	×
215	log::info!(".ef file found, using simplify split");	×
216
217	let graph =	×
218	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	×
219	.endianness::<E>()
220	.load()?;
221
222	thread_pool.install(\|\| {	×
223	let sorted = webgraph::transform::simplify_split(	×
224	&graph,	×
225	args.memory_usage.memory_usage,	×
226	)?;
227
NEW 228	builder.par_comp_lenders_endianness(&sorted, &target_endianness)	×
229	})?;
230
231	return Ok(());	×
232	}
233
234	no_ef_warn(&args.src);	×
235
236	let seq_graph =	×
237	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
238	.endianness::<E>()
239	.load()?;
240
241	// transpose the graph
242	let sorted =	×
243	webgraph::transform::simplify_sorted(seq_graph, args.memory_usage.memory_usage)?;	×
244
NEW 245	thread_pool	×
NEW 246	.install(\|\| builder.par_comp_lenders_endianness(&sorted, &target_endianness))?;	×
247	}
248	}
249
250	Ok(())	×
251	}

vigna / webgraph-rs / 22349485807

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous