21477217978

Committed 29 Jan 2026 11:55AM UTC coverage: 82.136% (+0.4%) from 81.767%

Build # 21477217978

Build Type

Pull #58

github

Committed by

BBertram-hex

Commit Message

57: cleanup

- fixed wrong error string
- clippy -> use slice instead of specific Container

Pull Request Pull Request #58: 57 new preprocessor step for whitlisting of CSV columns

Coverage Stats

70 of 81 new or added lines in 1 file covered. (86.42%)

35 existing lines in 2 files now uncovered.

2800 of 3409 relevant lines covered (82.14%)

2746.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.2

/src/csv/preprocessing.rs

use crate::csv;
use crate::csv::value::Value;
use crate::csv::Table;
use glob::Pattern;
use schemars_derive::JsonSchema;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering::Equal;
use tracing::{debug, info, warn};

#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
/// Preprocessor options
pub enum Preprocessor {
    /// Try to extract the headers from the first row - fallible if first row contains a number
    ExtractHeaders,
    /// Replace all fields in column by number by a deleted marker
    DeleteColumnByNumber(usize),
    /// Replace all fields in column by name by a deleted marker
    DeleteColumnByName(String),
    /// Replace with deleted marker: all fields in all columns that do not match any of the specified name
    KeepColumnsByName(Vec<String>),
    /// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
    SortByColumnName(String),
    /// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
    SortByColumnNumber(usize),
    /// Replace all fields in row with given number by a deleted marker
    DeleteRowByNumber(usize),
    /// Replace all fields in row  where at least a single field matches regex by a deleted marker
    DeleteRowByRegex(String),
    /// replace found cell using row and column index by a deleted marker
    DeleteCellByNumber {
        /// column number
        column: usize,
        /// row number
        row: usize,
    },
    /// replace found cell using column header and row index by a deleted marker
    DeleteCellByName {
        /// column with given name
        column: String,
        /// row number
        row: usize,
    },
}

impl Preprocessor {
    pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {
        match self {
            Preprocessor::ExtractHeaders => extract_headers(table),
            Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),
            Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),
            Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),
            Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),
            Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),
            Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),
            Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),
            Preprocessor::DeleteCellByNumber { column, row } => {
                delete_cell_by_number(table, *column, *row)
            }
            Preprocessor::DeleteCellByName { column, row } => {
                delete_cell_by_column_name_and_row_number(table, column, *row)
            }
        }
    }
}

fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {
    let regex = regex::Regex::new(regex)?;
    table
        .rows_mut()
        .filter(|row| row.iter().any(|v| regex.is_match(v.to_string().as_str())))
        .for_each(|mut row| row.iter_mut().for_each(|v| **v = Value::deleted()));
    Ok(())
}

fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(mut v) = table.rows_mut().nth(id) {
        v.iter_mut().for_each(|v| **v = Value::deleted())
    }
    Ok(())
}

fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {
    let value = table
        .columns
        .get_mut(column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn delete_cell_by_column_name_and_row_number(
    table: &mut Table,
    column: &str,
    row: usize,
) -> Result<(), csv::Error> {
    let value = table
        .columns
        .iter_mut()
        .find(|col| col.header.as_deref().unwrap_or_default() == column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {
    permutation::sort_by(rows_to_sort_by, |a, b| b.partial_cmp(a).unwrap_or(Equal))
}

fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {
    table.columns.iter_mut().for_each(|c| {
        permutation.apply_slice_in_place(&mut c.rows);
    });
}

fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    let sort_master_col = table.columns.get(id).ok_or_else(|| {
        csv::Error::InvalidAccess(format!(
            "Column number sorting by id {id} requested but column not found."
        ))
    })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column id".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let sort_master_col = table
        .columns
        .iter()
        .find(|c| c.header.as_deref().unwrap_or_default() == name)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!(
                "Requested format sorting by column'{name}' but column not found."
            ))
        })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column name".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let pattern = Pattern::new(name).map_err(|e| {
        csv::Error::InvalidAccess(format!(
            "Invalid glob pattern in DeleteColumnByName '{}': {}",
            name, e
        ))
    })?;

    if let Some(c) = table.columns.iter_mut().find(|col| {
        col.header.as_deref().unwrap_or_default() == name
            || pattern.matches(col.header.as_deref().unwrap_or_default())
    }) {
        c.delete_contents();
    }
    Ok(())
}

fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(col) = table.columns.get_mut(id) {
        col.delete_contents();
    }
    Ok(())
}

fn keep_columns_matching_any_names(table: &mut Table, names: &[String]) -> Result<(), csv::Error> {
    let patterns: Result<Vec<Pattern>, csv::Error> = names
        .iter()
        .map(|name| {
            Pattern::new(name).map_err(|e| {
                csv::Error::InvalidAccess(format!(
                    "Invalid glob pattern in KeepColumnsByName '{}': {}",
                    name, e
                ))
            })
        })
        .collect();
    let patterns = patterns?;

    table.columns.iter_mut().for_each(|col| {
        let header = col.header.as_deref().unwrap_or_default();
        if !(names.iter().zip(patterns.iter()).any(|(name, pattern)| {
            // Try exact match first, then glob pattern
            if header == name {
                info!("Keep header: \"{}\" (exact match)", header);
                true
            } else if pattern.matches(header) {
                info!("Keep header: \"{}\" (matches: \"{}\")", header, name);
                true
            } else {
                false
            }
        })) {
            // info!(
            //     "Discard column: \"{}\"",
            //     col.header.as_deref().unwrap_or_default()
            // );
            col.delete_contents();
        }
    });
    Ok(())
}

fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {
    debug!("Extracting headers...");
    let can_extract = table
        .columns
        .iter()
        .all(|c| matches!(c.rows.first(), Some(Value::String(_))));
    if !can_extract {
        warn!("Cannot extract header for this csv!");
        return Ok(());
    }

    for col in table.columns.iter_mut() {
        let title = col.rows.drain(0..1).next().ok_or_else(|| {
            csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())
        })?;
        if let Value::String(title) = title {
            col.header = Some(title);
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::csv::{Column, Delimiters, Error};
    use std::fs::File;

    macro_rules! string_vec {
        ($($x:expr),*) => (vec![$($x.to_string()),*]);
    }

    fn setup_table(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    fn setup_table_two(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/defects_headers.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    #[test]
    fn test_extract_headers_two() {
        let mut table = setup_table_two(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Entry"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Radius"
        );
    }

    #[test]
    fn test_extract_headers() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]"
        );
    }

    #[test]
    fn test_delete_column_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_number(&mut table, 0).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_column_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_name(&mut table, "Surface [mm²]").unwrap();
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_any_names() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Deviation [mm]",
            "Any name to be kept, which is no table header -> no effect"
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]",
            "First column was deleted, although it is in the list of names to be kept!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "Second column was not deleted, although it is not in the list of names to be kept!",
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_glob_patterns() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Surface*"  // Should match "Surface [mm²]" via glob pattern
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "First column (Deviation) should be deleted as it doesn't match pattern!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]",
            "Second column (Surface) was deleted, although it matches the glob pattern!",
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_row_by_id() {
        let mut table = setup_table(None);
        delete_row_by_number(&mut table, 0).unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_delete_row_by_regex() {
        let mut table = setup_table(None);
        delete_row_by_regex(&mut table, "mm").unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_sort_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        sort_by_column_name(&mut table, "Surface [mm²]").unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(1).unwrap().get_quantity().unwrap().value
                        >= next_row.get(1).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn test_sort_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        let column = 1;
        sort_by_column_id(&mut table, column).unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(column).unwrap().get_quantity().unwrap().value
                        >= next_row.get(column).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn sorting_by_mixed_column_fails() {
        let column = Column {
            header: Some("Field".to_string()),
            rows: vec![
                Value::from_str("1.0", &None),
                Value::String("String-Value".to_string()),
            ],
        };
        let mut table = Table {
            columns: vec![column],
        };
        let order_by_name = sort_by_column_name(&mut table, "Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));

        let order_by_id = sort_by_column_id(&mut table, 0);
        assert!(matches!(
            order_by_id.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));
    }

    #[test]
    fn non_existing_table_fails() {
        let mut table = setup_table(None);
        let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::InvalidAccess(_)
        ));

        let order_by_id = sort_by_column_id(&mut table, 999);
        assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));
    }

    #[test]
    fn test_delete_cell_by_numb() {
        let mut table = setup_table(None);
        delete_cell_by_number(&mut table, 1, 2).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(2)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_ne!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }

    #[test]
    fn test_delete_cell_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(3)
                .unwrap()
                .get_string(),
            None
        );

        assert_eq!(
            table
                .columns
                .get(0)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }
}

1	use crate::csv;
2	use crate::csv::value::Value;
3	use crate::csv::Table;
4	use glob::Pattern;
5	use schemars_derive::JsonSchema;
6	use serde::{Deserialize, Serialize};
7	use std::cmp::Ordering::Equal;
8	use tracing::{debug, info, warn};
9
10	#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
11	/// Preprocessor options
12	pub enum Preprocessor {
13	/// Try to extract the headers from the first row - fallible if first row contains a number
14	ExtractHeaders,
15	/// Replace all fields in column by number by a deleted marker
16	DeleteColumnByNumber(usize),
17	/// Replace all fields in column by name by a deleted marker
18	DeleteColumnByName(String),
19	/// Replace with deleted marker: all fields in all columns that do not match any of the specified name
20	KeepColumnsByName(Vec<String>),
21	/// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
22	SortByColumnName(String),
23	/// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
24	SortByColumnNumber(usize),
25	/// Replace all fields in row with given number by a deleted marker
26	DeleteRowByNumber(usize),
27	/// Replace all fields in row where at least a single field matches regex by a deleted marker
28	DeleteRowByRegex(String),
29	/// replace found cell using row and column index by a deleted marker
30	DeleteCellByNumber {
31	/// column number
32	column: usize,
33	/// row number
34	row: usize,
35	},
36	/// replace found cell using column header and row index by a deleted marker
37	DeleteCellByName {
38	/// column with given name
39	column: String,
40	/// row number
41	row: usize,
42	},
43	}
44
45	impl Preprocessor {
46	pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {	146✔
47	match self {	146✔
48	Preprocessor::ExtractHeaders => extract_headers(table),	146✔
UNCOV 49	Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),	×
UNCOV 50	Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),	×
NEW UNCOV 51	Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),	×
UNCOV 52	Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),	×
53	Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),	×
54	Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),	×
55	Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),	×
56	Preprocessor::DeleteCellByNumber { column, row } => {	×
57	delete_cell_by_number(table, column, row)	×
58	}
59	Preprocessor::DeleteCellByName { column, row } => {	×
60	delete_cell_by_column_name_and_row_number(table, column, *row)	×
61	}
62	}
63	}	146✔
64	}
65
66	fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {	1✔
67	let regex = regex::Regex::new(regex)?;	1✔
68	table	1✔
69	.rows_mut()	1✔
70	.filter(\|row\| row.iter().any(\|v\| regex.is_match(v.to_string().as_str())))	1,027✔
71	.for_each(\|mut row\| row.iter_mut().for_each(\|v\| **v = Value::deleted()));	2✔
72	Ok(())	1✔
73	}	1✔
74
75	fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
76	if let Some(mut v) = table.rows_mut().nth(id) {	1✔
77	v.iter_mut().for_each(\|v\| **v = Value::deleted())	2✔
UNCOV 78	}	×
79	Ok(())	1✔
80	}	1✔
81
82	fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {	1✔
83	let value = table	1✔
84	.columns	1✔
85	.get_mut(column)	1✔
86	.ok_or_else(\|\| {	1✔
UNCOV 87	csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))	×
88	})?	×
89	.rows
90	.get_mut(row)	1✔
91	.ok_or_else(\|\| {	1✔
UNCOV 92	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
UNCOV 93	})?;	×
94
95	*value = Value::deleted();	1✔
96
97	Ok(())	1✔
98	}	1✔
99
100	fn delete_cell_by_column_name_and_row_number(	1✔
101	table: &mut Table,	1✔
102	column: &str,	1✔
103	row: usize,	1✔
104	) -> Result<(), csv::Error> {	1✔
105	let value = table	1✔
106	.columns	1✔
107	.iter_mut()	1✔
108	.find(\|col\| col.header.as_deref().unwrap_or_default() == column)	2✔
109	.ok_or_else(\|\| {	1✔
UNCOV 110	csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))	×
UNCOV 111	})?	×
112	.rows
113	.get_mut(row)	1✔
114	.ok_or_else(\|\| {	1✔
UNCOV 115	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
UNCOV 116	})?;	×
117
118	*value = Value::deleted();	1✔
119
120	Ok(())	1✔
121	}	1✔
122
123	fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {	2✔
124	permutation::sort_by(rows_to_sort_by, \|a, b\| b.partial_cmp(a).unwrap_or(Equal))	6,862✔
125	}	2✔
126
127	fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {	2✔
128	table.columns.iter_mut().for_each(\|c\| {	4✔
129	permutation.apply_slice_in_place(&mut c.rows);	4✔
130	});	4✔
131	}	2✔
132
133	fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {	3✔
134	let sort_master_col = table.columns.get(id).ok_or_else(\|\| {	3✔
135	csv::Error::InvalidAccess(format!(	1✔
136	"Column number sorting by id {id} requested but column not found."	1✔
137	))	1✔
138	})?;	1✔
139	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
140	.rows	2✔
141	.iter()	2✔
142	.map(\|v\| {	515✔
143	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
144	csv::Error::UnexpectedValue(	1✔
145	v.clone(),	1✔
146	"Expected quantity while trying to sort by column id".to_string(),	1✔
147	)	1✔
148	})	1✔
149	})	515✔
150	.collect();	2✔
151	let permutation = get_permutation(&col_floats?);	2✔
152	apply_permutation(table, permutation);	1✔
153	Ok(())	1✔
154	}	3✔
155
156	fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	3✔
157	let sort_master_col = table	3✔
158	.columns	3✔
159	.iter()	3✔
160	.find(\|c\| c.header.as_deref().unwrap_or_default() == name)	5✔
161	.ok_or_else(\|\| {	3✔
162	csv::Error::InvalidAccess(format!(	1✔
163	"Requested format sorting by column'{name}' but column not found."	1✔
164	))	1✔
165	})?;	1✔
166	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
167	.rows	2✔
168	.iter()	2✔
169	.map(\|v\| {	515✔
170	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
171	csv::Error::UnexpectedValue(	1✔
172	v.clone(),	1✔
173	"Expected quantity while trying to sort by column name".to_string(),	1✔
174	)	1✔
175	})	1✔
176	})	515✔
177	.collect();	2✔
178	let permutation = get_permutation(&col_floats?);	2✔
179	apply_permutation(table, permutation);	1✔
180	Ok(())	1✔
181	}	3✔
182
183	fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	1✔
184	let pattern = Pattern::new(name).map_err(\|e\| {	1✔
NEW UNCOV 185	csv::Error::InvalidAccess(format!(	×
NEW UNCOV 186	"Invalid glob pattern in DeleteColumnByName '{}': {}",	×
NEW UNCOV 187	name, e	×
NEW UNCOV 188	))	×
NEW UNCOV 189	})?;	×
190
191	if let Some(c) = table.columns.iter_mut().find(\|col\| {	2✔
192	col.header.as_deref().unwrap_or_default() == name	2✔
193	\|\| pattern.matches(col.header.as_deref().unwrap_or_default())	1✔
194	}) {	2✔
195	c.delete_contents();	1✔
196	}	1✔
197	Ok(())	1✔
198	}	1✔
199
200	fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
201	if let Some(col) = table.columns.get_mut(id) {	1✔
202	col.delete_contents();	1✔
203	}	1✔
204	Ok(())	1✔
205	}	1✔
206
207	fn keep_columns_matching_any_names(table: &mut Table, names: &[String]) -> Result<(), csv::Error> {	2✔
208	let patterns: Result<Vec<Pattern>, csv::Error> = names	2✔
209	.iter()	2✔
210	.map(\|name\| {	3✔
211	Pattern::new(name).map_err(\|e\| {	3✔
NEW 212	csv::Error::InvalidAccess(format!(	×
NEW 213	"Invalid glob pattern in KeepColumnsByName '{}': {}",	×
NEW 214	name, e	×
NEW 215	))	×
NEW 216	})	×
217	})	3✔
218	.collect();	2✔
219	let patterns = patterns?;	2✔
220
221	table.columns.iter_mut().for_each(\|col\| {	4✔
222	let header = col.header.as_deref().unwrap_or_default();	4✔
223	if !(names.iter().zip(patterns.iter()).any(\|(name, pattern)\| {	5✔
224	// Try exact match first, then glob pattern
225	if header == name {	5✔
226	info!("Keep header: \"{}\" (exact match)", header);	1✔
227	true	1✔
228	} else if pattern.matches(header) {	4✔
229	info!("Keep header: \"{}\" (matches: \"{}\")", header, name);	1✔
230	true	1✔
231	} else {
232	false	3✔
233	}
234	})) {	5✔
235	// info!(	2✔
236	// "Discard column: \"{}\"",	2✔
237	// col.header.as_deref().unwrap_or_default()	2✔
238	// );	2✔
239	col.delete_contents();	2✔
240	}	2✔
241	});	4✔
242	Ok(())	2✔
243	}	2✔
244
245	fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {	155✔
246	debug!("Extracting headers...");	155✔
247	let can_extract = table	155✔
248	.columns	155✔
249	.iter()	155✔
250	.all(\|c\| matches!(c.rows.first(), Some(Value::String(_))));	396✔
251	if !can_extract {	155✔
252	warn!("Cannot extract header for this csv!");	×
253	return Ok(());	×
254	}	155✔
255
256	for col in table.columns.iter_mut() {	396✔
257	let title = col.rows.drain(0..1).next().ok_or_else(\|\| {	396✔
258	csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())	×
259	})?;	×
260	if let Value::String(title) = title {	396✔
261	col.header = Some(title);	396✔
262	}	396✔
263	}
264	Ok(())	155✔
265	}	155✔
266
267	#[cfg(test)]
268	mod tests {
269	use super::*;
270	use crate::csv::{Column, Delimiters, Error};
271	use std::fs::File;
272
273	macro_rules! string_vec {
274	($($x:expr),) => (vec![$($x.to_string()),]);
275	}
276
277	fn setup_table(delimiters: Option<Delimiters>) -> Table {	12✔
278	let delimiters = delimiters.unwrap_or_default();	12✔
279	Table::from_reader(	12✔
280	File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),	12✔
281	&delimiters,	12✔
282	)
283	.unwrap()	12✔
284	}	12✔
285
286	fn setup_table_two(delimiters: Option<Delimiters>) -> Table {	1✔
287	let delimiters = delimiters.unwrap_or_default();	1✔
288	Table::from_reader(	1✔
289	File::open("tests/csv/data/defects_headers.csv").unwrap(),	1✔
290	&delimiters,	1✔
291	)
292	.unwrap()	1✔
293	}	1✔
294
295	#[test]
296	fn test_extract_headers_two() {	1✔
297	let mut table = setup_table_two(None);	1✔
298	extract_headers(&mut table).unwrap();	1✔
299	assert_eq!(	1✔
300	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
301	"Entry"
302	);
303	assert_eq!(	1✔
304	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
305	"Radius"
306	);
307	}	1✔
308
309	#[test]
310	fn test_extract_headers() {	1✔
311	let mut table = setup_table(None);	1✔
312	extract_headers(&mut table).unwrap();	1✔
313	assert_eq!(	1✔
314	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
315	"Deviation [mm]"
316	);
317	assert_eq!(	1✔
318	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
319	"Surface [mm²]"
320	);
321	}	1✔
322
323	#[test]
324	fn test_delete_column_by_id() {	1✔
325	let mut table = setup_table(None);	1✔
326	extract_headers(&mut table).unwrap();	1✔
327	delete_column_number(&mut table, 0).unwrap();	1✔
328	assert_eq!(	1✔
329	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
330	"DELETED"
331	);
332	assert!(table	1✔
333	.columns	1✔
334	.first()	1✔
335	.unwrap()	1✔
336	.rows	1✔
337	.iter()	1✔
338	.all(\|v\| *v == Value::deleted()));	513✔
339	}	1✔
340
341	#[test]
342	fn test_delete_column_by_name() {	1✔
343	let mut table = setup_table(None);	1✔
344	extract_headers(&mut table).unwrap();	1✔
345	delete_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
346	assert_eq!(	1✔
347	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
348	"DELETED"
349	);
350	assert!(table	1✔
351	.columns	1✔
352	.last()	1✔
353	.unwrap()	1✔
354	.rows	1✔
355	.iter()	1✔
356	.all(\|v\| *v == Value::deleted()));	513✔
357	}	1✔
358
359	#[test]
360	fn test_keep_columns_matching_any_names() {	1✔
361	let mut table = setup_table(None);	1✔
362	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
363	extract_headers(&mut table).unwrap();	1✔
364
365	let keep_names = string_vec![	1✔
366	"Deviation [mm]",	1✔
367	"Any name to be kept, which is no table header -> no effect"	1✔
368	];
369	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
370	assert_eq!(	1✔
371	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
372	"Deviation [mm]",
373	"First column was deleted, although it is in the list of names to be kept!"
374	);
375	assert_eq!(	1✔
376	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
377	"DELETED",
378	"Second column was not deleted, although it is not in the list of names to be kept!",
379	);
380	assert!(table	1✔
381	.columns	1✔
382	.last()	1✔
383	.unwrap()	1✔
384	.rows	1✔
385	.iter()	1✔
386	.all(\|v\| *v == Value::deleted()));	513✔
387	}	1✔
388
389	#[test]
390	fn test_keep_columns_matching_glob_patterns() {	1✔
391	let mut table = setup_table(None);	1✔
392	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
393	extract_headers(&mut table).unwrap();	1✔
394
395	let keep_names = string_vec![	1✔
396	"Surface*" // Should match "Surface [mm²]" via glob pattern	1✔
397	];
398	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
399	assert_eq!(	1✔
400	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
401	"DELETED",
402	"First column (Deviation) should be deleted as it doesn't match pattern!"
403	);
404	assert_eq!(	1✔
405	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
406	"Surface [mm²]",
407	"Second column (Surface) was deleted, although it matches the glob pattern!",
408	);
409	assert!(table	1✔
410	.columns	1✔
411	.first()	1✔
412	.unwrap()	1✔
413	.rows	1✔
414	.iter()	1✔
415	.all(\|v\| *v == Value::deleted()));	513✔
416	}	1✔
417
418	#[test]
419	fn test_delete_row_by_id() {	1✔
420	let mut table = setup_table(None);	1✔
421	delete_row_by_number(&mut table, 0).unwrap();	1✔
422	assert_eq!(	1✔
423	table	1✔
424	.columns	1✔
425	.first()	1✔
426	.unwrap()	1✔
427	.rows	1✔
428	.first()	1✔
429	.unwrap()	1✔
430	.get_string()	1✔
431	.as_deref()	1✔
432	.unwrap(),	1✔
433	"DELETED"
434	);
435	}	1✔
436
437	#[test]
438	fn test_delete_row_by_regex() {	1✔
439	let mut table = setup_table(None);	1✔
440	delete_row_by_regex(&mut table, "mm").unwrap();	1✔
441	assert_eq!(	1✔
442	table	1✔
443	.columns	1✔
444	.first()	1✔
445	.unwrap()	1✔
446	.rows	1✔
447	.first()	1✔
448	.unwrap()	1✔
449	.get_string()	1✔
450	.as_deref()	1✔
451	.unwrap(),	1✔
452	"DELETED"
453	);
454	}	1✔
455
456	#[test]
457	fn test_sort_by_name() {	1✔
458	let mut table = setup_table(None);	1✔
459	extract_headers(&mut table).unwrap();	1✔
460	sort_by_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
461	let mut peekable_rows = table.rows().peekable();	1✔
462	while let Some(row) = peekable_rows.next() {	514✔
463	if let Some(next_row) = peekable_rows.peek() {	513✔
464	assert!(	512✔
465	row.get(1).unwrap().get_quantity().unwrap().value	512✔
466	>= next_row.get(1).unwrap().get_quantity().unwrap().value	512✔
467	);
468	}	1✔
469	}
470	}	1✔
471
472	#[test]
473	fn test_sort_by_id() {	1✔
474	let mut table = setup_table(None);	1✔
475	extract_headers(&mut table).unwrap();	1✔
476	let column = 1;	1✔
477	sort_by_column_id(&mut table, column).unwrap();	1✔
478	let mut peekable_rows = table.rows().peekable();	1✔
479	while let Some(row) = peekable_rows.next() {	514✔
480	if let Some(next_row) = peekable_rows.peek() {	513✔
481	assert!(	512✔
482	row.get(column).unwrap().get_quantity().unwrap().value	512✔
483	>= next_row.get(column).unwrap().get_quantity().unwrap().value	512✔
484	);
485	}	1✔
486	}
487	}	1✔
488
489	#[test]
490	fn sorting_by_mixed_column_fails() {	1✔
491	let column = Column {	1✔
492	header: Some("Field".to_string()),	1✔
493	rows: vec![	1✔
494	Value::from_str("1.0", &None),	1✔
495	Value::String("String-Value".to_string()),	1✔
496	],	1✔
497	};	1✔
498	let mut table = Table {	1✔
499	columns: vec![column],	1✔
500	};	1✔
501	let order_by_name = sort_by_column_name(&mut table, "Field");	1✔
502	assert!(matches!(	1✔
503	order_by_name.unwrap_err(),	1✔
504	Error::UnexpectedValue(_, _)
505	));
506
507	let order_by_id = sort_by_column_id(&mut table, 0);	1✔
508	assert!(matches!(	1✔
509	order_by_id.unwrap_err(),	1✔
510	Error::UnexpectedValue(_, _)
511	));
512	}	1✔
513
514	#[test]
515	fn non_existing_table_fails() {	1✔
516	let mut table = setup_table(None);	1✔
517	let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");	1✔
518	assert!(matches!(	1✔
519	order_by_name.unwrap_err(),	1✔
520	Error::InvalidAccess(_)
521	));
522
523	let order_by_id = sort_by_column_id(&mut table, 999);	1✔
524	assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));	1✔
525	}	1✔
526
527	#[test]
528	fn test_delete_cell_by_numb() {	1✔
529	let mut table = setup_table(None);	1✔
530	delete_cell_by_number(&mut table, 1, 2).unwrap();	1✔
531
532	assert_eq!(	1✔
533	table	1✔
534	.columns	1✔
535	.get(1)	1✔
536	.unwrap()	1✔
537	.rows	1✔
538	.get(2)	1✔
539	.unwrap()	1✔
540	.get_string()	1✔
541	.as_deref()	1✔
542	.unwrap(),	1✔
543	"DELETED"
544	);
545
546	assert_ne!(	1✔
547	table	1✔
548	.columns	1✔
549	.get(1)	1✔
550	.unwrap()	1✔
551	.rows	1✔
552	.first()	1✔
553	.unwrap()	1✔
554	.get_string()	1✔
555	.as_deref()	1✔
556	.unwrap(),	1✔
557	"DELETED"
558	);
559
560	assert_eq!(	1✔
561	table	1✔
562	.columns	1✔
563	.first()	1✔
564	.unwrap()	1✔
565	.rows	1✔
566	.get(1)	1✔
567	.unwrap()	1✔
568	.get_string(),	1✔
569	None
570	);
571	}	1✔
572
573	#[test]
574	fn test_delete_cell_by_name() {	1✔
575	let mut table = setup_table(None);	1✔
576	extract_headers(&mut table).unwrap();	1✔
577	delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();	1✔
578
579	assert_eq!(	1✔
580	table	1✔
581	.columns	1✔
582	.get(1)	1✔
583	.unwrap()	1✔
584	.rows	1✔
585	.get(1)	1✔
586	.unwrap()	1✔
587	.get_string()	1✔
588	.as_deref()	1✔
589	.unwrap(),	1✔
590	"DELETED"
591	);
592
593	assert_eq!(	1✔
594	table	1✔
595	.columns	1✔
596	.get(1)	1✔
597	.unwrap()	1✔
598	.rows	1✔
599	.get(3)	1✔
600	.unwrap()	1✔
601	.get_string(),	1✔
602	None
603	);
604
605	assert_eq!(	1✔
606	table	1✔
607	.columns	1✔
608	.get(0)	1✔
609	.unwrap()	1✔
610	.rows	1✔
611	.get(1)	1✔
612	.unwrap()	1✔
613	.get_string(),	1✔
614	None
615	);
616	}	1✔
617	}

VolumeGraphics / havocompare / 21477217978

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous