18761474399

Committed 23 Oct 2025 08:43PM UTC coverage: 61.976% (-0.06%) from 62.035%

Build # 18761474399

Build Type

push

github

Committed by

vigna

Commit Message

Goodbye le_bins/be_bins

Run Details

5201 of 8392 relevant lines covered (61.98%)

27888380.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/cli/src/transform/transpose.rs

/*
 * SPDX-FileCopyrightText: 2023 Inria
 * SPDX-FileCopyrightText: 2023 Tommaso Fontana
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */

use crate::*;
use anyhow::Result;
use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
use dsi_bitstream::prelude::*;
use std::io::BufReader;
use std::path::PathBuf;
use tempfile::Builder;
use webgraph::prelude::*;

#[derive(Parser, Debug)]
#[command(name = "transpose", about = "Transposes a BvGraph.", long_about = None)]
pub struct CliArgs {
    /// The basename of the graph.
    pub src: PathBuf,
    /// The basename of the transposed graph.
    pub dst: PathBuf,

    #[arg(short, long)]
    /// Use the parallel compressor.
    pub parallel: bool,

    #[clap(flatten)]
    pub num_threads: NumThreadsArg,

    #[clap(flatten)]
    pub memory_usage: MemoryUsageArg,

    #[clap(flatten)]
    pub ca: CompressArgs,
}

pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {
    create_parent_dir(&args.dst)?;

    match get_endianness(&args.src)?.as_str() {
        BE::NAME => {
            if args.parallel {
                par_transpose::<BE>(global_args, args)
            } else {
                transpose::<BE>(global_args, args)
            }
        }
        LE::NAME => {
            if args.parallel {
                par_transpose::<LE>(global_args, args)
            } else {
                transpose::<LE>(global_args, args)
            }
        }
        e => panic!("Unknown endianness: {}", e),
    }
}

pub fn transpose<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
{
    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);

    // TODO!: speed it up by using random access graph if possible
    let seq_graph = webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)
        .endianness::<E>()
        .load()?;

    // transpose the graph
    let sorted =
        webgraph::transform::transpose(&seq_graph, args.memory_usage.memory_usage).unwrap();

    let target_endianness = args.ca.endianness.clone();
    let dir = Builder::new().prefix("transform_transpose_").tempdir()?;
    BvComp::parallel_endianness(
        &args.dst,
        &sorted,
        sorted.num_nodes(),
        args.ca.into(),
        &thread_pool,
        dir,
        &target_endianness.unwrap_or_else(|| E::NAME.into()),
    )?;

    Ok(())
}

pub fn par_transpose<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>
where
    MmapHelper<u32>: CodesReaderFactoryHelper<E>,
    for<'a> <MmapHelper<u32> as CodesReaderFactory<E>>::CodesReader<'a>:
        BitSeek + Clone + Send + Sync,
    BufBitReader<E, WordAdapter<u32, BufReader<std::fs::File>>>: BitRead<E>,
    BufBitWriter<E, WordAdapter<usize, BufWriter<std::fs::File>>>: CodesWrite<E>,
{
    let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);

    let seq_graph = webgraph::graphs::bvgraph::BvGraph::with_basename(&args.src)
        .endianness::<E>()
        .load()?;

    // transpose the graph
    let split = webgraph::transform::transpose_split(&seq_graph, args.memory_usage.memory_usage)?;

    // Convert to (node, lender) pairs
    let pairs: Vec<_> = split.into();

    let dir = Builder::new().prefix("transform_transpose_").tempdir()?;
    BvComp::parallel_iter::<E, _>(
        &args.dst,
        pairs.into_iter().map(|lender| lender),
        seq_graph.num_nodes(),
        args.ca.into(),
        &thread_pool,
        dir,
    )?;
    Ok(())
}

1	/*
2	* SPDX-FileCopyrightText: 2023 Inria
3	* SPDX-FileCopyrightText: 2023 Tommaso Fontana
4	*
5	* SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
6	*/
7
8	use crate::*;
9	use anyhow::Result;
10	use dsi_bitstream::dispatch::factory::CodesReaderFactoryHelper;
11	use dsi_bitstream::prelude::*;
12	use std::io::BufReader;
13	use std::path::PathBuf;
14	use tempfile::Builder;
15	use webgraph::prelude::*;
16
17	#[derive(Parser, Debug)]
18	#[command(name = "transpose", about = "Transposes a BvGraph.", long_about = None)]
19	pub struct CliArgs {
20	/// The basename of the graph.
21	pub src: PathBuf,
22	/// The basename of the transposed graph.
23	pub dst: PathBuf,
24
25	#[arg(short, long)]
26	/// Use the parallel compressor.
27	pub parallel: bool,
28
29	#[clap(flatten)]
30	pub num_threads: NumThreadsArg,
31
32	#[clap(flatten)]
33	pub memory_usage: MemoryUsageArg,
34
35	#[clap(flatten)]
36	pub ca: CompressArgs,
37	}
38
39	pub fn main(global_args: GlobalArgs, args: CliArgs) -> Result<()> {	×
40	create_parent_dir(&args.dst)?;	×
41
42	match get_endianness(&args.src)?.as_str() {	×
43	BE::NAME => {	×
44	if args.parallel {	×
45	par_transpose::<BE>(global_args, args)	×
46	} else {
47	transpose::<BE>(global_args, args)	×
48	}
49	}
50	LE::NAME => {	×
51	if args.parallel {	×
52	par_transpose::<LE>(global_args, args)	×
53	} else {
54	transpose::<LE>(global_args, args)	×
55	}
56	}
57	e => panic!("Unknown endianness: {}", e),	×
58	}
59	}
60
61	pub fn transpose<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>	×
62	where
63	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
64	{
65	let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);	×
66
67	// TODO!: speed it up by using random access graph if possible
68	let seq_graph = webgraph::graphs::bvgraph::sequential::BvGraphSeq::with_basename(&args.src)	×
69	.endianness::<E>()
70	.load()?;
71
72	// transpose the graph
73	let sorted =	×
74	webgraph::transform::transpose(&seq_graph, args.memory_usage.memory_usage).unwrap();	×
75
76	let target_endianness = args.ca.endianness.clone();	×
77	let dir = Builder::new().prefix("transform_transpose_").tempdir()?;	×
78	BvComp::parallel_endianness(
79	&args.dst,	×
80	&sorted,	×
81	sorted.num_nodes(),	×
82	args.ca.into(),	×
83	&thread_pool,	×
84	dir,	×
85	&target_endianness.unwrap_or_else(\|\| E::NAME.into()),	×
86	)?;
87
88	Ok(())	×
89	}
90
91	pub fn par_transpose<E: Endianness>(_global_args: GlobalArgs, args: CliArgs) -> Result<()>	×
92	where
93	MmapHelper<u32>: CodesReaderFactoryHelper<E>,
94	for<'a> <MmapHelper<u32> as CodesReaderFactory<E>>::CodesReader<'a>:
95	BitSeek + Clone + Send + Sync,
96	BufBitReader<E, WordAdapter<u32, BufReader<std::fs::File>>>: BitRead<E>,
97	BufBitWriter<E, WordAdapter<usize, BufWriter<std::fs::File>>>: CodesWrite<E>,
98	{
99	let thread_pool = crate::get_thread_pool(args.num_threads.num_threads);	×
100
101	let seq_graph = webgraph::graphs::bvgraph::BvGraph::with_basename(&args.src)	×
102	.endianness::<E>()
103	.load()?;
104
105	// transpose the graph
106	let split = webgraph::transform::transpose_split(&seq_graph, args.memory_usage.memory_usage)?;	×
107
108	// Convert to (node, lender) pairs
109	let pairs: Vec<_> = split.into();	×
110
111	let dir = Builder::new().prefix("transform_transpose_").tempdir()?;	×
112	BvComp::parallel_iter::<E, _>(
113	&args.dst,	×
114	pairs.into_iter().map(\|lender\| lender),	×
115	seq_graph.num_nodes(),	×
116	args.ca.into(),	×
117	&thread_pool,	×
118	dir,	×
119	)?;
120	Ok(())	×
121	}

vigna / webgraph-rs / 18761474399

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous