14196927616

Committed 01 Apr 2025 01:28PM UTC coverage: 56.787% (+7.4%) from 49.345%

Build # 14196927616

Build Type

Pull #122

github

Committed by

web-flow

Commit Message

Merge 706583c76 into 5b5393a19

Pull Request Pull Request #122: Refactored argmin and argmax into an extension trait for Iterator and moved argmin_filtered and argmax_filtered into compute.rs

Run Details

1279 of 1630 new or added lines in 34 files covered. (78.47%)

3736 of 6579 relevant lines covered (56.79%)

13936227.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

11.94

/cli/src/transform/simplify.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use crate::*;
use anyhow::Result;
use dsi_bitstream::{dispatch::factory::CodesReaderFactoryHelper, prelude::*};
use mmap_rs::MmapFlags;
use std::path::PathBuf;
use tempfile::Builder;
use webgraph::graphs::union_graph::UnionGraph;
use webgraph::prelude::*;

#[derive(Parser, Debug)]
#[command(name = "simplify", about = "Makes a BvGraph simple (undirected and loopless) by adding missing arcs and removing loops, optionally applying a permutation.", long_about = None)]
pub struct CliArgs {
    /// The basename of the graph.
    pub src: PathBuf,
    /// The basename of the simplified graph.
    pub dst: PathBuf,

    #[arg(long)]
    /// The basename of a pre-computed transposed version of the source graph, which
    /// will be use to speed up the simplification.
    pub transposed: Option<PathBuf>,

    #[clap(flatten)]
    pub num_threads: NumThreadsArg,

    #[clap(flatten)]
    pub batch_size: BatchSizeArg,

    #[clap(flatten)]
    pub ca: CompressArgs,

    #[arg(long)]
    /// The path to an optional permutation in binary big-endian format to apply to the graph.
    pub permutation: Option<PathBuf>,
}

pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
    create_parent_dir(&args.dst)?;

    match get_endianness(&args.src)?.as_str() {
        #[cfg(any(
            feature = "be_bins",
            not(any(feature = "be_bins", feature = "le_bins"))
        ))]
        BE::NAME => simplify::<BE>(global_args, args),
        #[cfg(any(
            feature = "le_bins",
            not(any(feature = "be_bins", feature = "le_bins"))
        ))]
        LE::NAME => simplify::<LE>(global_args, args),
        e => panic!("Unknown endianness: {}", e),
    }
}

fn no_ef_warn(basepath: impl AsRef<std::path::Path>) {
    log::warn!(
        "The .ef file was not found so the simplification will proceed sequentially. This may be slow. To speed it up, you can use `webgraph build ef {}` which would allow us create batches in parallel",
        basepath.as_ref().display()
    );
}

pub fn simplify<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
    for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Clone + Send + Sync,
{
    // TODO!: speed it up by using random access graph if possible
    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);

    let target_endianness = args.ca.endianness.clone().unwrap_or_else(|| E::NAME.into());

    let dir = Builder::new().prefix("transform_simplify_").tempdir()?;

    match (args.permutation, args.transposed) {
        // load the transposed graph and use it to directly compress the graph
        // without doing any sorting
        (None, Some(t_path)) => {
            log::info!("Transposed graph provided, using it to simplify the graph");

            let has_ef_graph =
                std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file());
            let has_ef_t_graph =
                std::fs::metadata(t_path.with_extension("ef")).is_ok_and(|x| x.is_file());

            match (has_ef_graph, has_ef_t_graph) {
                (true, true) => {
                    log::info!("Both .ef files found, using simplify split");

                    let graph =
                        webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                            .endianness::<E>()
                            .load()?;
                    let num_nodes = graph.num_nodes();
                    let graph_t =
                        webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&t_path)
                            .endianness::<E>()
                            .load()?;

                    if graph_t.num_nodes() != num_nodes {
                        anyhow::bail!(
                            "The number of nodes in the graph and its transpose do not match! {} != {}",
                            num_nodes,
                            graph_t.num_nodes()
                        );
                    }

                    let sorted = NoSelfLoopsGraph(UnionGraph(graph, graph_t));

                    BvComp::parallel_endianness(
                        &args.dst,
                        &sorted,
                        num_nodes,
                        args.ca.into(),
                        &thread_pool,
                        dir,
                        &target_endianness,
                    )?;

                    return Ok(());
                }
                (true, false) => {
                    no_ef_warn(&args.src);
                }
                (false, true) => {
                    no_ef_warn(&t_path);
                }
                (false, false) => {
                    no_ef_warn(&args.src);
                    no_ef_warn(&t_path);
                }
            }

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;
            let num_nodes = seq_graph.num_nodes();
            let seq_graph_t =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&t_path)
                    .endianness::<E>()
                    .load()?;

            if seq_graph_t.num_nodes() != num_nodes {
                anyhow::bail!(
                    "The number of nodes in the graph and its transpose do not match! {} != {}",
                    num_nodes,
                    seq_graph_t.num_nodes()
                );
            }

            let sorted = NoSelfLoopsGraph(UnionGraph(seq_graph, seq_graph_t));

            BvComp::parallel_endianness(
                &args.dst,
                &sorted,
                num_nodes,
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness,
            )?;
        }
        // apply the permutation, don't care if the transposed graph is already computed
        // as we cannot really exploit it
        (Some(perm_path), None | Some(_)) => {
            log::info!("Permutation provided, applying it to the graph");

            let perm = JavaPermutation::mmap(perm_path, MmapFlags::RANDOM_ACCESS)?;

            // if the .ef file exists, we can use the simplify split
            if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file()) {
                log::info!(".ef file found, using simplify split");
                let graph =
                    webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                        .endianness::<E>()
                        .load()?;

                let perm_graph = PermutedGraph {
                    graph: &graph,
                    perm: &perm,
                };

                let sorted = webgraph::transform::simplify_split(
                    &perm_graph,
                    args.batch_size.batch_size,
                    &thread_pool,
                )?;

                BvComp::parallel_endianness(
                    &args.dst,
                    &sorted,
                    graph.num_nodes(),
                    args.ca.into(),
                    &thread_pool,
                    dir,
                    &target_endianness,
                )?;

                return Ok(());
            }

            no_ef_warn(&args.src);

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;

            let perm_graph = PermutedGraph {
                graph: &seq_graph,
                perm: &perm,
            };

            // simplify the graph
            let sorted =
                webgraph::transform::simplify(&perm_graph, args.batch_size.batch_size).unwrap();

            BvComp::parallel_endianness(
                &args.dst,
                &sorted,
                sorted.num_nodes(),
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness,
            )?;
        }
        // just compute the transpose on the fly
        (None, None) => {
            log::info!(
                "No permutation or transposed graph provided, computing the transpose on the fly"
            );
            // if the .ef file exists, we can use the simplify split
            if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(|x| x.is_file()) {
                log::info!(".ef file found, using simplify split");

                let graph =
                    webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)
                        .endianness::<E>()
                        .load()?;

                let sorted = webgraph::transform::simplify_split(
                    &graph,
                    args.batch_size.batch_size,
                    &thread_pool,
                )?;

                BvComp::parallel_endianness(
                    &args.dst,
                    &sorted,
                    graph.num_nodes(),
                    args.ca.into(),
                    &thread_pool,
                    dir,
                    &target_endianness,
                )?;

                return Ok(());
            }

            no_ef_warn(&args.src);

            let seq_graph =
                webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
                    .endianness::<E>()
                    .load()?;

            let num_nodes = seq_graph.num_nodes();
            // transpose the graph
            let sorted =
                webgraph::transform::simplify_sorted(seq_graph, args.batch_size.batch_size)
                    .unwrap();

            BvComp::parallel_endianness(
                &args.dst,
                &sorted,
                num_nodes,
                args.ca.into(),
                &thread_pool,
                dir,
                &target_endianness,
            )?;
        }
    }

    Ok(())
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Tommaso Fontana
4	*
5	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6	*/
7
8	use crate::*;
9	use anyhow::Result;
10	use dsi_bitstream::{dispatch::factory::CodesReaderFactoryHelper, prelude::*};
11	use mmap_rs::MmapFlags;
12	use std::path::PathBuf;
13	use tempfile::Builder;
14	use webgraph::graphs::union_graph::UnionGraph;
15	use webgraph::prelude::*;
16
17	#[derive(Parser, Debug)]
18	#[command(name = "simplify", about = "Makes a BvGraph simple (undirected and loopless) by adding missing arcs and removing loops, optionally applying a permutation.", long_about = None)]
19	pub struct CliArgs {
20	/// The basename of the graph.
21	pub src: PathBuf,
22	/// The basename of the simplified graph.
23	pub dst: PathBuf,
24
25	#[arg(long)]
26	/// The basename of a pre-computed transposed version of the source graph, which
27	/// will be use to speed up the simplification.
28	pub transposed: Option<PathBuf>,
29
30	#[clap(flatten)]
31	pub num_threads: NumThreadsArg,
32
33	#[clap(flatten)]
34	pub batch_size: BatchSizeArg,
35
36	#[clap(flatten)]
37	pub ca: CompressArgs,
38
39	#[arg(long)]
40	/// The path to an optional permutation in binary big-endian format to apply to the graph.
41	pub permutation: Option<PathBuf>,
42	}
43
44	pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {	2✔
45	create_parent_dir(&args.dst)?;	2✔
46
47	match get_endianness(&args.src)?.as_str() {	4✔
48	#[cfg(any(
49	feature = "be_bins",
50	not(any(feature = "be_bins", feature = "le_bins"))
51	))]
52	BE::NAME => simplify::<BE>(global_args, args),	4✔
53	#[cfg(any(
54	feature = "le_bins",
55	not(any(feature = "be_bins", feature = "le_bins"))
56	))]
57	LE::NAME => simplify::<LE>(global_args, args),	×
58	e => panic!("Unknown endianness: {}", e),	×
59	}
60	}
61
62	fn no_ef_warn(basepath: impl AsRef<std::path::Path>) {	×
NEW 63	log::warn!(	×
NEW 64	"The .ef file was not found so the simplification will proceed sequentially. This may be slow. To speed it up, you can use `webgraph build ef {}` which would allow us create batches in parallel",	×
NEW 65	basepath.as_ref().display()	×
66	);
67	}
68
69	pub fn simplify<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>	2✔
70	where
71	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
72	for<'a> LoadModeCodesReader<'a, E, Mmap>: BitSeek + Clone + Send + Sync,
73	{
74	// TODO!: speed it up by using random access graph if possible
75	let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);	2✔
76
77	let target_endianness = args.ca.endianness.clone().unwrap_or_else(\|\| E::NAME.into());	6✔
78
79	let dir = Builder::new().prefix("transform_simplify_").tempdir()?;	4✔
80
81	match (args.permutation, args.transposed) {	×
82	// load the transposed graph and use it to directly compress the graph
83	// without doing any sorting
84	(None, Some(t_path)) => {	×
85	log::info!("Transposed graph provided, using it to simplify the graph");	×
86
87	let has_ef_graph =	×
88	std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file());	×
89	let has_ef_t_graph =	×
90	std::fs::metadata(t_path.with_extension("ef")).is_ok_and(\|x\| x.is_file());	×
91
92	match (has_ef_graph, has_ef_t_graph) {	×
93	(true, true) => {	×
94	log::info!("Both .ef files found, using simplify split");	×
95
96	let graph =	×
NEW 97	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	×
98	.endianness::<E>()
99	.load()?;
100	let num_nodes = graph.num_nodes();	×
101	let graph_t =	×
NEW 102	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&t_path)	×
103	.endianness::<E>()
104	.load()?;
105
106	if graph_t.num_nodes() != num_nodes {	×
NEW 107	anyhow::bail!(	×
NEW 108	"The number of nodes in the graph and its transpose do not match! {} != {}",	×
NEW 109	num_nodes,	×
NEW 110	graph_t.num_nodes()	×
111	);
112	}
113
114	let sorted = NoSelfLoopsGraph(UnionGraph(graph, graph_t));	×
115
116	BvComp::parallel_endianness(
117	&args.dst,	×
118	&sorted,	×
119	num_nodes,	×
120	args.ca.into(),	×
121	&thread_pool,	×
122	dir,	×
123	&target_endianness,	×
124	)?;
125
126	return Ok(());	×
127	}
128	(true, false) => {	×
129	no_ef_warn(&args.src);	×
130	}
131	(false, true) => {	×
132	no_ef_warn(&t_path);	×
133	}
134	(false, false) => {	×
135	no_ef_warn(&args.src);	×
136	no_ef_warn(&t_path);	×
137	}
138	}
139
140	let seq_graph =	×
NEW 141	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
142	.endianness::<E>()
143	.load()?;
144	let num_nodes = seq_graph.num_nodes();	×
145	let seq_graph_t =	×
NEW 146	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&t_path)	×
147	.endianness::<E>()
148	.load()?;
149
150	if seq_graph_t.num_nodes() != num_nodes {	×
151	anyhow::bail!(	×
152	"The number of nodes in the graph and its transpose do not match! {} != {}",	×
153	num_nodes,	×
154	seq_graph_t.num_nodes()	×
155	);
156	}
157
158	let sorted = NoSelfLoopsGraph(UnionGraph(seq_graph, seq_graph_t));	×
159
160	BvComp::parallel_endianness(
161	&args.dst,	×
162	&sorted,	×
163	num_nodes,	×
164	args.ca.into(),	×
165	&thread_pool,	×
166	dir,	×
167	&target_endianness,	×
168	)?;
169	}
170	// apply the permutation, don't care if the transposed graph is already computed
171	// as we cannot really exploit it
172	(Some(perm_path), None \| Some(_)) => {	2✔
173	log::info!("Permutation provided, applying it to the graph");	4✔
174
175	let perm = JavaPermutation::mmap(perm_path, MmapFlags::RANDOM_ACCESS)?;	4✔
176
177	// if the .ef file exists, we can use the simplify split
178	if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file()) {	2✔
179	log::info!(".ef file found, using simplify split");	4✔
180	let graph =	2✔
181	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	2✔
182	.endianness::<E>()
183	.load()?;
184
185	let perm_graph = PermutedGraph {
186	graph: &graph,	×
187	perm: &perm,	×
188	};
189
190	let sorted = webgraph::transform::simplify_split(
191	&perm_graph,	×
192	args.batch_size.batch_size,	×
193	&thread_pool,	×
194	)?;
195
196	BvComp::parallel_endianness(
197	&args.dst,	×
198	&sorted,	×
199	graph.num_nodes(),	×
200	args.ca.into(),	×
201	&thread_pool,	×
202	dir,	×
203	&target_endianness,	×
204	)?;
205
206	return Ok(());	2✔
207	}
208
209	no_ef_warn(&args.src);	×
210
211	let seq_graph =	×
NEW 212	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
213	.endianness::<E>()
214	.load()?;
215
216	let perm_graph = PermutedGraph {
217	graph: &seq_graph,	×
218	perm: &perm,	×
219	};
220
221	// simplify the graph
222	let sorted =	×
NEW 223	webgraph::transform::simplify(&perm_graph, args.batch_size.batch_size).unwrap();	×
224
225	BvComp::parallel_endianness(
226	&args.dst,	×
227	&sorted,	×
228	sorted.num_nodes(),	×
229	args.ca.into(),	×
230	&thread_pool,	×
231	dir,	×
232	&target_endianness,	×
233	)?;
234	}
235	// just compute the transpose on the fly
236	(None, None) => {	×
237	log::info!(	×
238	"No permutation or transposed graph provided, computing the transpose on the fly"	×
239	);
240	// if the .ef file exists, we can use the simplify split
241	if std::fs::metadata(args.src.with_extension("ef")).is_ok_and(\|x\| x.is_file()) {	×
242	log::info!(".ef file found, using simplify split");	×
243
244	let graph =	×
NEW 245	webgraph::graphs::bvgraph::random_access::BvGraph::with_basename(&args.src)	×
246	.endianness::<E>()
247	.load()?;
248
249	let sorted = webgraph::transform::simplify_split(
250	&graph,	×
251	args.batch_size.batch_size,	×
252	&thread_pool,	×
253	)?;
254
255	BvComp::parallel_endianness(
256	&args.dst,	×
257	&sorted,	×
258	graph.num_nodes(),	×
259	args.ca.into(),	×
260	&thread_pool,	×
261	dir,	×
262	&target_endianness,	×
263	)?;
264
265	return Ok(());	×
266	}
267
268	no_ef_warn(&args.src);	×
269
270	let seq_graph =	×
NEW 271	webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
272	.endianness::<E>()
273	.load()?;
274
275	let num_nodes = seq_graph.num_nodes();	×
276	// transpose the graph
277	let sorted =	×
NEW 278	webgraph::transform::simplify_sorted(seq_graph, args.batch_size.batch_size)	×
279	.unwrap();
280
281	BvComp::parallel_endianness(
282	&args.dst,	×
283	&sorted,	×
284	num_nodes,	×
285	args.ca.into(),	×
286	&thread_pool,	×
287	dir,	×
288	&target_endianness,	×
289	)?;
290	}
291	}
292
293	Ok(())	×
294	}

vigna / webgraph-rs / 14196927616

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous