21437489529

Committed 28 Jan 2026 12:02PM UTC coverage: 82.272% (+0.2%) from 82.058%

Build # 21437489529

Build Type

push

github

Committed by

BBertram-hex

Commit Message

57: Whitelisting with KeepColumnByName accepts glob patterns

Run Details

39 of 45 new or added lines in 1 file covered. (86.67%)

14 existing lines in 1 file now uncovered.

2803 of 3407 relevant lines covered (82.27%)

2747.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.32

/src/csv/preprocessing.rs

use crate::csv;
use crate::csv::value::Value;
use crate::csv::Table;
use glob::Pattern;
use schemars_derive::JsonSchema;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering::Equal;
use tracing::{debug, info, warn};

#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
/// Preprocessor options
pub enum Preprocessor {
    /// Try to extract the headers from the first row - fallible if first row contains a number
    ExtractHeaders,
    /// Replace all fields in column by number by a deleted marker
    DeleteColumnByNumber(usize),
    /// Replace all fields in column by name by a deleted marker
    DeleteColumnByName(String),
    /// Replace with deleted marker: all fields in all columns that do not match any of the specified name
    KeepColumnsByName(Vec<String>),
    /// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
    SortByColumnName(String),
    /// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
    SortByColumnNumber(usize),
    /// Replace all fields in row with given number by a deleted marker
    DeleteRowByNumber(usize),
    /// Replace all fields in row  where at least a single field matches regex by a deleted marker
    DeleteRowByRegex(String),
    /// replace found cell using row and column index by a deleted marker
    DeleteCellByNumber {
        /// column number
        column: usize,
        /// row number
        row: usize,
    },
    /// replace found cell using column header and row index by a deleted marker
    DeleteCellByName {
        /// column with given name
        column: String,
        /// row number
        row: usize,
    },
}

impl Preprocessor {
    pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {
        match self {
            Preprocessor::ExtractHeaders => extract_headers(table),
            Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),
            Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),
            Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),
            Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),
            Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),
            Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),
            Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),
            Preprocessor::DeleteCellByNumber { column, row } => {
                delete_cell_by_number(table, *column, *row)
            }
            Preprocessor::DeleteCellByName { column, row } => {
                delete_cell_by_column_name_and_row_number(table, column, *row)
            }
        }
    }
}

fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {
    let regex = regex::Regex::new(regex)?;
    table
        .rows_mut()
        .filter(|row| row.iter().any(|v| regex.is_match(v.to_string().as_str())))
        .for_each(|mut row| row.iter_mut().for_each(|v| **v = Value::deleted()));
    Ok(())
}

fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(mut v) = table.rows_mut().nth(id) {
        v.iter_mut().for_each(|v| **v = Value::deleted())
    }
    Ok(())
}

fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {
    let value = table
        .columns
        .get_mut(column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn delete_cell_by_column_name_and_row_number(
    table: &mut Table,
    column: &str,
    row: usize,
) -> Result<(), csv::Error> {
    let value = table
        .columns
        .iter_mut()
        .find(|col| col.header.as_deref().unwrap_or_default() == column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {
    permutation::sort_by(rows_to_sort_by, |a, b| b.partial_cmp(a).unwrap_or(Equal))
}

fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {
    table.columns.iter_mut().for_each(|c| {
        permutation.apply_slice_in_place(&mut c.rows);
    });
}

fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    let sort_master_col = table.columns.get(id).ok_or_else(|| {
        csv::Error::InvalidAccess(format!(
            "Column number sorting by id {id} requested but column not found."
        ))
    })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column id".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let sort_master_col = table
        .columns
        .iter()
        .find(|c| c.header.as_deref().unwrap_or_default() == name)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!(
                "Requested format sorting by column'{name}' but column not found."
            ))
        })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column name".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    if let Some(c) = table
        .columns
        .iter_mut()
        .find(|col| col.header.as_deref().unwrap_or_default() == name)
    {
        c.delete_contents();
    }
    Ok(())
}

fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(col) = table.columns.get_mut(id) {
        col.delete_contents();
    }
    Ok(())
}

fn keep_columns_matching_any_names(
    table: &mut Table,
    names: &Vec<String>,
) -> Result<(), csv::Error> {
    let patterns: Result<Vec<Pattern>, csv::Error> = names
        .iter()
        .map(|name| {
            Pattern::new(name).map_err(|e| {
                csv::Error::InvalidAccess(format!(
                    "Invalid glob pattern in KeepColumnsByName '{}': {}",
                    name, e
                ))
            })
        })
        .collect();
    let patterns = patterns?;

    table.columns.iter_mut().for_each(|col| {
        let header = col.header.as_deref().unwrap_or_default();
        if !(names.iter().zip(patterns.iter()).any(|(name, pattern)| {
            // Try exact match first, then glob pattern
            if header == name {
                info!("Keep header: \"{}\" (exact match)", header);
                true
            } else if pattern.matches(header) {
                info!("Keep header: \"{}\" (matches: \"{}\")", header, name);
                true
            } else {
                false
            }
        })) {
            // info!(
            //     "Discard column: \"{}\"",
            //     col.header.as_deref().unwrap_or_default()
            // );
            col.delete_contents();
        }
    });
    Ok(())
}

fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {
    debug!("Extracting headers...");
    let can_extract = table
        .columns
        .iter()
        .all(|c| matches!(c.rows.first(), Some(Value::String(_))));
    if !can_extract {
        warn!("Cannot extract header for this csv!");
        return Ok(());
    }

    for col in table.columns.iter_mut() {
        let title = col.rows.drain(0..1).next().ok_or_else(|| {
            csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())
        })?;
        if let Value::String(title) = title {
            col.header = Some(title);
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::csv::{Column, Delimiters, Error};
    use std::fs::File;

    macro_rules! string_vec {
        ($($x:expr),*) => (vec![$($x.to_string()),*]);
    }

    fn setup_table(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    fn setup_table_two(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/defects_headers.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    #[test]
    fn test_extract_headers_two() {
        let mut table = setup_table_two(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Entry"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Radius"
        );
    }

    #[test]
    fn test_extract_headers() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]"
        );
    }

    #[test]
    fn test_delete_column_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_number(&mut table, 0).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_column_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_name(&mut table, "Surface [mm²]").unwrap();
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_any_names() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Deviation [mm]",
            "Any name to be kept, which is no table header -> no effect"
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]",
            "First column was deleted, although it is in the list of names to be kept!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "Second column was not deleted, although it is not in the list of names to be kept!",
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_glob_patterns() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Surface*"  // Should match "Surface [mm²]" via glob pattern
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "First column (Deviation) should be deleted as it doesn't match pattern!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]",
            "Second column (Surface) was deleted, although it matches the glob pattern!",
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_row_by_id() {
        let mut table = setup_table(None);
        delete_row_by_number(&mut table, 0).unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_delete_row_by_regex() {
        let mut table = setup_table(None);
        delete_row_by_regex(&mut table, "mm").unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_sort_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        sort_by_column_name(&mut table, "Surface [mm²]").unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(1).unwrap().get_quantity().unwrap().value
                        >= next_row.get(1).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn test_sort_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        let column = 1;
        sort_by_column_id(&mut table, column).unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(column).unwrap().get_quantity().unwrap().value
                        >= next_row.get(column).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn sorting_by_mixed_column_fails() {
        let column = Column {
            header: Some("Field".to_string()),
            rows: vec![
                Value::from_str("1.0", &None),
                Value::String("String-Value".to_string()),
            ],
        };
        let mut table = Table {
            columns: vec![column],
        };
        let order_by_name = sort_by_column_name(&mut table, "Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));

        let order_by_id = sort_by_column_id(&mut table, 0);
        assert!(matches!(
            order_by_id.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));
    }

    #[test]
    fn non_existing_table_fails() {
        let mut table = setup_table(None);
        let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::InvalidAccess(_)
        ));

        let order_by_id = sort_by_column_id(&mut table, 999);
        assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));
    }

    #[test]
    fn test_delete_cell_by_numb() {
        let mut table = setup_table(None);
        delete_cell_by_number(&mut table, 1, 2).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(2)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_ne!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }

    #[test]
    fn test_delete_cell_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(3)
                .unwrap()
                .get_string(),
            None
        );

        assert_eq!(
            table
                .columns
                .get(0)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }
}

1	use crate::csv;
2	use crate::csv::value::Value;
3	use crate::csv::Table;
4	use glob::Pattern;
5	use schemars_derive::JsonSchema;
6	use serde::{Deserialize, Serialize};
7	use std::cmp::Ordering::Equal;
8	use tracing::{debug, info, warn};
9
10	#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
11	/// Preprocessor options
12	pub enum Preprocessor {
13	/// Try to extract the headers from the first row - fallible if first row contains a number
14	ExtractHeaders,
15	/// Replace all fields in column by number by a deleted marker
16	DeleteColumnByNumber(usize),
17	/// Replace all fields in column by name by a deleted marker
18	DeleteColumnByName(String),
19	/// Replace with deleted marker: all fields in all columns that do not match any of the specified name
20	KeepColumnsByName(Vec<String>),
21	/// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
22	SortByColumnName(String),
23	/// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
24	SortByColumnNumber(usize),
25	/// Replace all fields in row with given number by a deleted marker
26	DeleteRowByNumber(usize),
27	/// Replace all fields in row where at least a single field matches regex by a deleted marker
28	DeleteRowByRegex(String),
29	/// replace found cell using row and column index by a deleted marker
30	DeleteCellByNumber {
31	/// column number
32	column: usize,
33	/// row number
34	row: usize,
35	},
36	/// replace found cell using column header and row index by a deleted marker
37	DeleteCellByName {
38	/// column with given name
39	column: String,
40	/// row number
41	row: usize,
42	},
43	}
44
45	impl Preprocessor {
46	pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {	146✔
47	match self {	146✔
48	Preprocessor::ExtractHeaders => extract_headers(table),	146✔
49	Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),	×
50	Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),	×
NEW 51	Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),	×
52	Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),	×
UNCOV 53	Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),	×
54	Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),	×
55	Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),	×
UNCOV 56	Preprocessor::DeleteCellByNumber { column, row } => {	×
UNCOV 57	delete_cell_by_number(table, column, row)	×
58	}
UNCOV 59	Preprocessor::DeleteCellByName { column, row } => {	×
UNCOV 60	delete_cell_by_column_name_and_row_number(table, column, *row)	×
61	}
62	}
63	}	146✔
64	}
65
66	fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {	1✔
67	let regex = regex::Regex::new(regex)?;	1✔
68	table	1✔
69	.rows_mut()	1✔
70	.filter(\|row\| row.iter().any(\|v\| regex.is_match(v.to_string().as_str())))	1,027✔
71	.for_each(\|mut row\| row.iter_mut().for_each(\|v\| **v = Value::deleted()));	2✔
72	Ok(())	1✔
73	}	1✔
74
75	fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
76	if let Some(mut v) = table.rows_mut().nth(id) {	1✔
77	v.iter_mut().for_each(\|v\| **v = Value::deleted())	2✔
UNCOV 78	}	×
79	Ok(())	1✔
80	}	1✔
81
82	fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {	1✔
83	let value = table	1✔
84	.columns	1✔
85	.get_mut(column)	1✔
86	.ok_or_else(\|\| {	1✔
87	csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))	×
88	})?	×
89	.rows
90	.get_mut(row)	1✔
91	.ok_or_else(\|\| {	1✔
UNCOV 92	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
UNCOV 93	})?;	×
94
95	*value = Value::deleted();	1✔
96
97	Ok(())	1✔
98	}	1✔
99
100	fn delete_cell_by_column_name_and_row_number(	1✔
101	table: &mut Table,	1✔
102	column: &str,	1✔
103	row: usize,	1✔
104	) -> Result<(), csv::Error> {	1✔
105	let value = table	1✔
106	.columns	1✔
107	.iter_mut()	1✔
108	.find(\|col\| col.header.as_deref().unwrap_or_default() == column)	2✔
109	.ok_or_else(\|\| {	1✔
110	csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))	×
111	})?	×
112	.rows
113	.get_mut(row)	1✔
114	.ok_or_else(\|\| {	1✔
UNCOV 115	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
UNCOV 116	})?;	×
117
118	*value = Value::deleted();	1✔
119
120	Ok(())	1✔
121	}	1✔
122
123	fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {	2✔
124	permutation::sort_by(rows_to_sort_by, \|a, b\| b.partial_cmp(a).unwrap_or(Equal))	6,862✔
125	}	2✔
126
127	fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {	2✔
128	table.columns.iter_mut().for_each(\|c\| {	4✔
129	permutation.apply_slice_in_place(&mut c.rows);	4✔
130	});	4✔
131	}	2✔
132
133	fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {	3✔
134	let sort_master_col = table.columns.get(id).ok_or_else(\|\| {	3✔
135	csv::Error::InvalidAccess(format!(	1✔
136	"Column number sorting by id {id} requested but column not found."	1✔
137	))	1✔
138	})?;	1✔
139	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
140	.rows	2✔
141	.iter()	2✔
142	.map(\|v\| {	515✔
143	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
144	csv::Error::UnexpectedValue(	1✔
145	v.clone(),	1✔
146	"Expected quantity while trying to sort by column id".to_string(),	1✔
147	)	1✔
148	})	1✔
149	})	515✔
150	.collect();	2✔
151	let permutation = get_permutation(&col_floats?);	2✔
152	apply_permutation(table, permutation);	1✔
153	Ok(())	1✔
154	}	3✔
155
156	fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	3✔
157	let sort_master_col = table	3✔
158	.columns	3✔
159	.iter()	3✔
160	.find(\|c\| c.header.as_deref().unwrap_or_default() == name)	5✔
161	.ok_or_else(\|\| {	3✔
162	csv::Error::InvalidAccess(format!(	1✔
163	"Requested format sorting by column'{name}' but column not found."	1✔
164	))	1✔
165	})?;	1✔
166	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
167	.rows	2✔
168	.iter()	2✔
169	.map(\|v\| {	515✔
170	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
171	csv::Error::UnexpectedValue(	1✔
172	v.clone(),	1✔
173	"Expected quantity while trying to sort by column name".to_string(),	1✔
174	)	1✔
175	})	1✔
176	})	515✔
177	.collect();	2✔
178	let permutation = get_permutation(&col_floats?);	2✔
179	apply_permutation(table, permutation);	1✔
180	Ok(())	1✔
181	}	3✔
182
183	fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	1✔
184	if let Some(c) = table	1✔
185	.columns	1✔
186	.iter_mut()	1✔
187	.find(\|col\| col.header.as_deref().unwrap_or_default() == name)	2✔
188	{	1✔
189	c.delete_contents();	1✔
190	}	1✔
191	Ok(())	1✔
192	}	1✔
193
194	fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
195	if let Some(col) = table.columns.get_mut(id) {	1✔
196	col.delete_contents();	1✔
197	}	1✔
198	Ok(())	1✔
199	}	1✔
200
201	fn keep_columns_matching_any_names(	2✔
202	table: &mut Table,	2✔
203	names: &Vec<String>,	2✔
204	) -> Result<(), csv::Error> {	2✔
205	let patterns: Result<Vec<Pattern>, csv::Error> = names	2✔
206	.iter()	2✔
207	.map(\|name\| {	3✔
208	Pattern::new(name).map_err(\|e\| {	3✔
NEW 209	csv::Error::InvalidAccess(format!(	×
NEW 210	"Invalid glob pattern in KeepColumnsByName '{}': {}",	×
NEW 211	name, e	×
NEW 212	))	×
NEW 213	})	×
214	})	3✔
215	.collect();	2✔
216	let patterns = patterns?;	2✔
217
218	table.columns.iter_mut().for_each(\|col\| {	4✔
219	let header = col.header.as_deref().unwrap_or_default();	4✔
220	if !(names.iter().zip(patterns.iter()).any(\|(name, pattern)\| {	5✔
221	// Try exact match first, then glob pattern
222	if header == name {	5✔
223	info!("Keep header: \"{}\" (exact match)", header);	1✔
224	true	1✔
225	} else if pattern.matches(header) {	4✔
226	info!("Keep header: \"{}\" (matches: \"{}\")", header, name);	1✔
227	true	1✔
228	} else {
229	false	3✔
230	}
231	})) {	5✔
232	// info!(	2✔
233	// "Discard column: \"{}\"",	2✔
234	// col.header.as_deref().unwrap_or_default()	2✔
235	// );	2✔
236	col.delete_contents();	2✔
237	}	2✔
238	});	4✔
239	Ok(())	2✔
240	}	2✔
241
242	fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {	155✔
243	debug!("Extracting headers...");	155✔
244	let can_extract = table	155✔
245	.columns	155✔
246	.iter()	155✔
247	.all(\|c\| matches!(c.rows.first(), Some(Value::String(_))));	396✔
248	if !can_extract {	155✔
UNCOV 249	warn!("Cannot extract header for this csv!");	×
UNCOV 250	return Ok(());	×
251	}	155✔
252
253	for col in table.columns.iter_mut() {	396✔
254	let title = col.rows.drain(0..1).next().ok_or_else(\|\| {	396✔
UNCOV 255	csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())	×
UNCOV 256	})?;	×
257	if let Value::String(title) = title {	396✔
258	col.header = Some(title);	396✔
259	}	396✔
260	}
261	Ok(())	155✔
262	}	155✔
263
264	#[cfg(test)]
265	mod tests {
266	use super::*;
267	use crate::csv::{Column, Delimiters, Error};
268	use std::fs::File;
269
270	macro_rules! string_vec {
271	($($x:expr),) => (vec![$($x.to_string()),]);
272	}
273
274	fn setup_table(delimiters: Option<Delimiters>) -> Table {	12✔
275	let delimiters = delimiters.unwrap_or_default();	12✔
276	Table::from_reader(	12✔
277	File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),	12✔
278	&delimiters,	12✔
279	)
280	.unwrap()	12✔
281	}	12✔
282
283	fn setup_table_two(delimiters: Option<Delimiters>) -> Table {	1✔
284	let delimiters = delimiters.unwrap_or_default();	1✔
285	Table::from_reader(	1✔
286	File::open("tests/csv/data/defects_headers.csv").unwrap(),	1✔
287	&delimiters,	1✔
288	)
289	.unwrap()	1✔
290	}	1✔
291
292	#[test]
293	fn test_extract_headers_two() {	1✔
294	let mut table = setup_table_two(None);	1✔
295	extract_headers(&mut table).unwrap();	1✔
296	assert_eq!(	1✔
297	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
298	"Entry"
299	);
300	assert_eq!(	1✔
301	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
302	"Radius"
303	);
304	}	1✔
305
306	#[test]
307	fn test_extract_headers() {	1✔
308	let mut table = setup_table(None);	1✔
309	extract_headers(&mut table).unwrap();	1✔
310	assert_eq!(	1✔
311	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
312	"Deviation [mm]"
313	);
314	assert_eq!(	1✔
315	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
316	"Surface [mm²]"
317	);
318	}	1✔
319
320	#[test]
321	fn test_delete_column_by_id() {	1✔
322	let mut table = setup_table(None);	1✔
323	extract_headers(&mut table).unwrap();	1✔
324	delete_column_number(&mut table, 0).unwrap();	1✔
325	assert_eq!(	1✔
326	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
327	"DELETED"
328	);
329	assert!(table	1✔
330	.columns	1✔
331	.first()	1✔
332	.unwrap()	1✔
333	.rows	1✔
334	.iter()	1✔
335	.all(\|v\| *v == Value::deleted()));	513✔
336	}	1✔
337
338	#[test]
339	fn test_delete_column_by_name() {	1✔
340	let mut table = setup_table(None);	1✔
341	extract_headers(&mut table).unwrap();	1✔
342	delete_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
343	assert_eq!(	1✔
344	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
345	"DELETED"
346	);
347	assert!(table	1✔
348	.columns	1✔
349	.last()	1✔
350	.unwrap()	1✔
351	.rows	1✔
352	.iter()	1✔
353	.all(\|v\| *v == Value::deleted()));	513✔
354	}	1✔
355
356	#[test]
357	fn test_keep_columns_matching_any_names() {	1✔
358	let mut table = setup_table(None);	1✔
359	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
360	extract_headers(&mut table).unwrap();	1✔
361
362	let keep_names = string_vec![	1✔
363	"Deviation [mm]",	1✔
364	"Any name to be kept, which is no table header -> no effect"	1✔
365	];
366	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
367	assert_eq!(	1✔
368	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
369	"Deviation [mm]",
370	"First column was deleted, although it is in the list of names to be kept!"
371	);
372	assert_eq!(	1✔
373	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
374	"DELETED",
375	"Second column was not deleted, although it is not in the list of names to be kept!",
376	);
377	assert!(table	1✔
378	.columns	1✔
379	.last()	1✔
380	.unwrap()	1✔
381	.rows	1✔
382	.iter()	1✔
383	.all(\|v\| *v == Value::deleted()));	513✔
384	}	1✔
385
386	#[test]
387	fn test_keep_columns_matching_glob_patterns() {	1✔
388	let mut table = setup_table(None);	1✔
389	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
390	extract_headers(&mut table).unwrap();	1✔
391
392	let keep_names = string_vec![	1✔
393	"Surface*" // Should match "Surface [mm²]" via glob pattern	1✔
394	];
395	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
396	assert_eq!(	1✔
397	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
398	"DELETED",
399	"First column (Deviation) should be deleted as it doesn't match pattern!"
400	);
401	assert_eq!(	1✔
402	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
403	"Surface [mm²]",
404	"Second column (Surface) was deleted, although it matches the glob pattern!",
405	);
406	assert!(table	1✔
407	.columns	1✔
408	.first()	1✔
409	.unwrap()	1✔
410	.rows	1✔
411	.iter()	1✔
412	.all(\|v\| *v == Value::deleted()));	513✔
413	}	1✔
414
415	#[test]
416	fn test_delete_row_by_id() {	1✔
417	let mut table = setup_table(None);	1✔
418	delete_row_by_number(&mut table, 0).unwrap();	1✔
419	assert_eq!(	1✔
420	table	1✔
421	.columns	1✔
422	.first()	1✔
423	.unwrap()	1✔
424	.rows	1✔
425	.first()	1✔
426	.unwrap()	1✔
427	.get_string()	1✔
428	.as_deref()	1✔
429	.unwrap(),	1✔
430	"DELETED"
431	);
432	}	1✔
433
434	#[test]
435	fn test_delete_row_by_regex() {	1✔
436	let mut table = setup_table(None);	1✔
437	delete_row_by_regex(&mut table, "mm").unwrap();	1✔
438	assert_eq!(	1✔
439	table	1✔
440	.columns	1✔
441	.first()	1✔
442	.unwrap()	1✔
443	.rows	1✔
444	.first()	1✔
445	.unwrap()	1✔
446	.get_string()	1✔
447	.as_deref()	1✔
448	.unwrap(),	1✔
449	"DELETED"
450	);
451	}	1✔
452
453	#[test]
454	fn test_sort_by_name() {	1✔
455	let mut table = setup_table(None);	1✔
456	extract_headers(&mut table).unwrap();	1✔
457	sort_by_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
458	let mut peekable_rows = table.rows().peekable();	1✔
459	while let Some(row) = peekable_rows.next() {	514✔
460	if let Some(next_row) = peekable_rows.peek() {	513✔
461	assert!(	512✔
462	row.get(1).unwrap().get_quantity().unwrap().value	512✔
463	>= next_row.get(1).unwrap().get_quantity().unwrap().value	512✔
464	);
465	}	1✔
466	}
467	}	1✔
468
469	#[test]
470	fn test_sort_by_id() {	1✔
471	let mut table = setup_table(None);	1✔
472	extract_headers(&mut table).unwrap();	1✔
473	let column = 1;	1✔
474	sort_by_column_id(&mut table, column).unwrap();	1✔
475	let mut peekable_rows = table.rows().peekable();	1✔
476	while let Some(row) = peekable_rows.next() {	514✔
477	if let Some(next_row) = peekable_rows.peek() {	513✔
478	assert!(	512✔
479	row.get(column).unwrap().get_quantity().unwrap().value	512✔
480	>= next_row.get(column).unwrap().get_quantity().unwrap().value	512✔
481	);
482	}	1✔
483	}
484	}	1✔
485
486	#[test]
487	fn sorting_by_mixed_column_fails() {	1✔
488	let column = Column {	1✔
489	header: Some("Field".to_string()),	1✔
490	rows: vec![	1✔
491	Value::from_str("1.0", &None),	1✔
492	Value::String("String-Value".to_string()),	1✔
493	],	1✔
494	};	1✔
495	let mut table = Table {	1✔
496	columns: vec![column],	1✔
497	};	1✔
498	let order_by_name = sort_by_column_name(&mut table, "Field");	1✔
499	assert!(matches!(	1✔
500	order_by_name.unwrap_err(),	1✔
501	Error::UnexpectedValue(_, _)
502	));
503
504	let order_by_id = sort_by_column_id(&mut table, 0);	1✔
505	assert!(matches!(	1✔
506	order_by_id.unwrap_err(),	1✔
507	Error::UnexpectedValue(_, _)
508	));
509	}	1✔
510
511	#[test]
512	fn non_existing_table_fails() {	1✔
513	let mut table = setup_table(None);	1✔
514	let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");	1✔
515	assert!(matches!(	1✔
516	order_by_name.unwrap_err(),	1✔
517	Error::InvalidAccess(_)
518	));
519
520	let order_by_id = sort_by_column_id(&mut table, 999);	1✔
521	assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));	1✔
522	}	1✔
523
524	#[test]
525	fn test_delete_cell_by_numb() {	1✔
526	let mut table = setup_table(None);	1✔
527	delete_cell_by_number(&mut table, 1, 2).unwrap();	1✔
528
529	assert_eq!(	1✔
530	table	1✔
531	.columns	1✔
532	.get(1)	1✔
533	.unwrap()	1✔
534	.rows	1✔
535	.get(2)	1✔
536	.unwrap()	1✔
537	.get_string()	1✔
538	.as_deref()	1✔
539	.unwrap(),	1✔
540	"DELETED"
541	);
542
543	assert_ne!(	1✔
544	table	1✔
545	.columns	1✔
546	.get(1)	1✔
547	.unwrap()	1✔
548	.rows	1✔
549	.first()	1✔
550	.unwrap()	1✔
551	.get_string()	1✔
552	.as_deref()	1✔
553	.unwrap(),	1✔
554	"DELETED"
555	);
556
557	assert_eq!(	1✔
558	table	1✔
559	.columns	1✔
560	.first()	1✔
561	.unwrap()	1✔
562	.rows	1✔
563	.get(1)	1✔
564	.unwrap()	1✔
565	.get_string(),	1✔
566	None
567	);
568	}	1✔
569
570	#[test]
571	fn test_delete_cell_by_name() {	1✔
572	let mut table = setup_table(None);	1✔
573	extract_headers(&mut table).unwrap();	1✔
574	delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();	1✔
575
576	assert_eq!(	1✔
577	table	1✔
578	.columns	1✔
579	.get(1)	1✔
580	.unwrap()	1✔
581	.rows	1✔
582	.get(1)	1✔
583	.unwrap()	1✔
584	.get_string()	1✔
585	.as_deref()	1✔
586	.unwrap(),	1✔
587	"DELETED"
588	);
589
590	assert_eq!(	1✔
591	table	1✔
592	.columns	1✔
593	.get(1)	1✔
594	.unwrap()	1✔
595	.rows	1✔
596	.get(3)	1✔
597	.unwrap()	1✔
598	.get_string(),	1✔
599	None
600	);
601
602	assert_eq!(	1✔
603	table	1✔
604	.columns	1✔
605	.get(0)	1✔
606	.unwrap()	1✔
607	.rows	1✔
608	.get(1)	1✔
609	.unwrap()	1✔
610	.get_string(),	1✔
611	None
612	);
613	}	1✔
614	}

VolumeGraphics / havocompare / 21437489529

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous