21443895732

Committed 28 Jan 2026 03:16PM UTC coverage: 82.151% (+0.4%) from 81.767%

Build # 21443895732

Build Type

Pull #58

github

Committed by

web-flow

Commit Message

Merge 56512e472 into 3f79c1fdc

Pull Request Pull Request #58: 57 new preprocessor step for whitlisting of CSV columns

Run Details

73 of 84 new or added lines in 1 file covered. (86.9%)

4 existing lines in 1 file now uncovered.

2803 of 3412 relevant lines covered (82.15%)

2743.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.26

/src/csv/preprocessing.rs

use crate::csv;
use crate::csv::value::Value;
use crate::csv::Table;
use glob::Pattern;
use schemars_derive::JsonSchema;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering::Equal;
use tracing::{debug, info, warn};

#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
/// Preprocessor options
pub enum Preprocessor {
    /// Try to extract the headers from the first row - fallible if first row contains a number
    ExtractHeaders,
    /// Replace all fields in column by number by a deleted marker
    DeleteColumnByNumber(usize),
    /// Replace all fields in column by name by a deleted marker
    DeleteColumnByName(String),
    /// Replace with deleted marker: all fields in all columns that do not match any of the specified name
    KeepColumnsByName(Vec<String>),
    /// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
    SortByColumnName(String),
    /// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
    SortByColumnNumber(usize),
    /// Replace all fields in row with given number by a deleted marker
    DeleteRowByNumber(usize),
    /// Replace all fields in row  where at least a single field matches regex by a deleted marker
    DeleteRowByRegex(String),
    /// replace found cell using row and column index by a deleted marker
    DeleteCellByNumber {
        /// column number
        column: usize,
        /// row number
        row: usize,
    },
    /// replace found cell using column header and row index by a deleted marker
    DeleteCellByName {
        /// column with given name
        column: String,
        /// row number
        row: usize,
    },
}

impl Preprocessor {
    pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {
        match self {
            Preprocessor::ExtractHeaders => extract_headers(table),
            Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),
            Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),
            Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),
            Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),
            Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),
            Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),
            Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),
            Preprocessor::DeleteCellByNumber { column, row } => {
                delete_cell_by_number(table, *column, *row)
            }
            Preprocessor::DeleteCellByName { column, row } => {
                delete_cell_by_column_name_and_row_number(table, column, *row)
            }
        }
    }
}

fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {
    let regex = regex::Regex::new(regex)?;
    table
        .rows_mut()
        .filter(|row| row.iter().any(|v| regex.is_match(v.to_string().as_str())))
        .for_each(|mut row| row.iter_mut().for_each(|v| **v = Value::deleted()));
    Ok(())
}

fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(mut v) = table.rows_mut().nth(id) {
        v.iter_mut().for_each(|v| **v = Value::deleted())
    }
    Ok(())
}

fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {
    let value = table
        .columns
        .get_mut(column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn delete_cell_by_column_name_and_row_number(
    table: &mut Table,
    column: &str,
    row: usize,
) -> Result<(), csv::Error> {
    let value = table
        .columns
        .iter_mut()
        .find(|col| col.header.as_deref().unwrap_or_default() == column)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))
        })?
        .rows
        .get_mut(row)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))
        })?;

    *value = Value::deleted();

    Ok(())
}

fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {
    permutation::sort_by(rows_to_sort_by, |a, b| b.partial_cmp(a).unwrap_or(Equal))
}

fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {
    table.columns.iter_mut().for_each(|c| {
        permutation.apply_slice_in_place(&mut c.rows);
    });
}

fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    let sort_master_col = table.columns.get(id).ok_or_else(|| {
        csv::Error::InvalidAccess(format!(
            "Column number sorting by id {id} requested but column not found."
        ))
    })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column id".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let sort_master_col = table
        .columns
        .iter()
        .find(|c| c.header.as_deref().unwrap_or_default() == name)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!(
                "Requested format sorting by column'{name}' but column not found."
            ))
        })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column name".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let pattern = Pattern::new(name).map_err(|e| {
        csv::Error::InvalidAccess(format!(
            "Invalid glob pattern in KeepColumnsByName '{}': {}",
            name, e
        ))
    })?;

    if let Some(c) = table.columns.iter_mut().find(|col| {
        col.header.as_deref().unwrap_or_default() == name
            || pattern.matches(col.header.as_deref().unwrap_or_default())
    }) {
        c.delete_contents();
    }
    Ok(())
}

fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(col) = table.columns.get_mut(id) {
        col.delete_contents();
    }
    Ok(())
}

fn keep_columns_matching_any_names(
    table: &mut Table,
    names: &Vec<String>,
) -> Result<(), csv::Error> {
    let patterns: Result<Vec<Pattern>, csv::Error> = names
        .iter()
        .map(|name| {
            Pattern::new(name).map_err(|e| {
                csv::Error::InvalidAccess(format!(
                    "Invalid glob pattern in KeepColumnsByName '{}': {}",
                    name, e
                ))
            })
        })
        .collect();
    let patterns = patterns?;

    table.columns.iter_mut().for_each(|col| {
        let header = col.header.as_deref().unwrap_or_default();
        if !(names.iter().zip(patterns.iter()).any(|(name, pattern)| {
            // Try exact match first, then glob pattern
            if header == name {
                info!("Keep header: \"{}\" (exact match)", header);
                true
            } else if pattern.matches(header) {
                info!("Keep header: \"{}\" (matches: \"{}\")", header, name);
                true
            } else {
                false
            }
        })) {
            // info!(
            //     "Discard column: \"{}\"",
            //     col.header.as_deref().unwrap_or_default()
            // );
            col.delete_contents();
        }
    });
    Ok(())
}

fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {
    debug!("Extracting headers...");
    let can_extract = table
        .columns
        .iter()
        .all(|c| matches!(c.rows.first(), Some(Value::String(_))));
    if !can_extract {
        warn!("Cannot extract header for this csv!");
        return Ok(());
    }

    for col in table.columns.iter_mut() {
        let title = col.rows.drain(0..1).next().ok_or_else(|| {
            csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())
        })?;
        if let Value::String(title) = title {
            col.header = Some(title);
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::csv::{Column, Delimiters, Error};
    use std::fs::File;

    macro_rules! string_vec {
        ($($x:expr),*) => (vec![$($x.to_string()),*]);
    }

    fn setup_table(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    fn setup_table_two(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/defects_headers.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    #[test]
    fn test_extract_headers_two() {
        let mut table = setup_table_two(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Entry"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Radius"
        );
    }

    #[test]
    fn test_extract_headers() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]"
        );
    }

    #[test]
    fn test_delete_column_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_number(&mut table, 0).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_column_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_name(&mut table, "Surface [mm²]").unwrap();
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_any_names() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Deviation [mm]",
            "Any name to be kept, which is no table header -> no effect"
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]",
            "First column was deleted, although it is in the list of names to be kept!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "Second column was not deleted, although it is not in the list of names to be kept!",
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_keep_columns_matching_glob_patterns() {
        let mut table = setup_table(None);
        // column headers in test table: "Deviation [mm]", "Surface [mm²]"
        extract_headers(&mut table).unwrap();

        let keep_names = string_vec![
            "Surface*"  // Should match "Surface [mm²]" via glob pattern
        ];
        keep_columns_matching_any_names(&mut table, &keep_names).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED",
            "First column (Deviation) should be deleted as it doesn't match pattern!"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]",
            "Second column (Surface) was deleted, although it matches the glob pattern!",
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == Value::deleted()));
    }

    #[test]
    fn test_delete_row_by_id() {
        let mut table = setup_table(None);
        delete_row_by_number(&mut table, 0).unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_delete_row_by_regex() {
        let mut table = setup_table(None);
        delete_row_by_regex(&mut table, "mm").unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_sort_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        sort_by_column_name(&mut table, "Surface [mm²]").unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(1).unwrap().get_quantity().unwrap().value
                        >= next_row.get(1).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn test_sort_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        let column = 1;
        sort_by_column_id(&mut table, column).unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(column).unwrap().get_quantity().unwrap().value
                        >= next_row.get(column).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn sorting_by_mixed_column_fails() {
        let column = Column {
            header: Some("Field".to_string()),
            rows: vec![
                Value::from_str("1.0", &None),
                Value::String("String-Value".to_string()),
            ],
        };
        let mut table = Table {
            columns: vec![column],
        };
        let order_by_name = sort_by_column_name(&mut table, "Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));

        let order_by_id = sort_by_column_id(&mut table, 0);
        assert!(matches!(
            order_by_id.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));
    }

    #[test]
    fn non_existing_table_fails() {
        let mut table = setup_table(None);
        let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::InvalidAccess(_)
        ));

        let order_by_id = sort_by_column_id(&mut table, 999);
        assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));
    }

    #[test]
    fn test_delete_cell_by_numb() {
        let mut table = setup_table(None);
        delete_cell_by_number(&mut table, 1, 2).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(2)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_ne!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }

    #[test]
    fn test_delete_cell_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );

        assert_eq!(
            table
                .columns
                .get(1)
                .unwrap()
                .rows
                .get(3)
                .unwrap()
                .get_string(),
            None
        );

        assert_eq!(
            table
                .columns
                .get(0)
                .unwrap()
                .rows
                .get(1)
                .unwrap()
                .get_string(),
            None
        );
    }
}

1	use crate::csv;
2	use crate::csv::value::Value;
3	use crate::csv::Table;
4	use glob::Pattern;
5	use schemars_derive::JsonSchema;
6	use serde::{Deserialize, Serialize};
7	use std::cmp::Ordering::Equal;
8	use tracing::{debug, info, warn};
9
10	#[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)]
11	/// Preprocessor options
12	pub enum Preprocessor {
13	/// Try to extract the headers from the first row - fallible if first row contains a number
14	ExtractHeaders,
15	/// Replace all fields in column by number by a deleted marker
16	DeleteColumnByNumber(usize),
17	/// Replace all fields in column by name by a deleted marker
18	DeleteColumnByName(String),
19	/// Replace with deleted marker: all fields in all columns that do not match any of the specified name
20	KeepColumnsByName(Vec<String>),
21	/// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
22	SortByColumnName(String),
23	/// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
24	SortByColumnNumber(usize),
25	/// Replace all fields in row with given number by a deleted marker
26	DeleteRowByNumber(usize),
27	/// Replace all fields in row where at least a single field matches regex by a deleted marker
28	DeleteRowByRegex(String),
29	/// replace found cell using row and column index by a deleted marker
30	DeleteCellByNumber {
31	/// column number
32	column: usize,
33	/// row number
34	row: usize,
35	},
36	/// replace found cell using column header and row index by a deleted marker
37	DeleteCellByName {
38	/// column with given name
39	column: String,
40	/// row number
41	row: usize,
42	},
43	}
44
45	impl Preprocessor {
46	pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {	146✔
47	match self {	146✔
48	Preprocessor::ExtractHeaders => extract_headers(table),	146✔
49	Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),	×
50	Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),	×
NEW 51	Preprocessor::KeepColumnsByName(names) => keep_columns_matching_any_names(table, names),	×
52	Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),	×
53	Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),	×
54	Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),	×
55	Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),	×
56	Preprocessor::DeleteCellByNumber { column, row } => {	×
57	delete_cell_by_number(table, column, row)	×
58	}
59	Preprocessor::DeleteCellByName { column, row } => {	×
60	delete_cell_by_column_name_and_row_number(table, column, *row)	×
61	}
62	}
63	}	146✔
64	}
65
66	fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {	1✔
67	let regex = regex::Regex::new(regex)?;	1✔
68	table	1✔
69	.rows_mut()	1✔
70	.filter(\|row\| row.iter().any(\|v\| regex.is_match(v.to_string().as_str())))	1,027✔
71	.for_each(\|mut row\| row.iter_mut().for_each(\|v\| **v = Value::deleted()));	2✔
72	Ok(())	1✔
73	}	1✔
74
75	fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
76	if let Some(mut v) = table.rows_mut().nth(id) {	1✔
77	v.iter_mut().for_each(\|v\| **v = Value::deleted())	2✔
78	}	×
79	Ok(())	1✔
80	}	1✔
81
82	fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> {	1✔
83	let value = table	1✔
84	.columns	1✔
85	.get_mut(column)	1✔
86	.ok_or_else(\|\| {	1✔
87	csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column))	×
88	})?	×
89	.rows
90	.get_mut(row)	1✔
91	.ok_or_else(\|\| {	1✔
92	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
93	})?;	×
94
95	*value = Value::deleted();	1✔
96
97	Ok(())	1✔
98	}	1✔
99
100	fn delete_cell_by_column_name_and_row_number(	1✔
101	table: &mut Table,	1✔
102	column: &str,	1✔
103	row: usize,	1✔
104	) -> Result<(), csv::Error> {	1✔
105	let value = table	1✔
106	.columns	1✔
107	.iter_mut()	1✔
108	.find(\|col\| col.header.as_deref().unwrap_or_default() == column)	2✔
109	.ok_or_else(\|\| {	1✔
110	csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column))	×
111	})?	×
112	.rows
113	.get_mut(row)	1✔
114	.ok_or_else(\|\| {	1✔
115	csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row))	×
116	})?;	×
117
118	*value = Value::deleted();	1✔
119
120	Ok(())	1✔
121	}	1✔
122
123	fn get_permutation(rows_to_sort_by: &Vec<f64>) -> permutation::Permutation {	2✔
124	permutation::sort_by(rows_to_sort_by, \|a, b\| b.partial_cmp(a).unwrap_or(Equal))	6,862✔
125	}	2✔
126
127	fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {	2✔
128	table.columns.iter_mut().for_each(\|c\| {	4✔
129	permutation.apply_slice_in_place(&mut c.rows);	4✔
130	});	4✔
131	}	2✔
132
133	fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {	3✔
134	let sort_master_col = table.columns.get(id).ok_or_else(\|\| {	3✔
135	csv::Error::InvalidAccess(format!(	1✔
136	"Column number sorting by id {id} requested but column not found."	1✔
137	))	1✔
138	})?;	1✔
139	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
140	.rows	2✔
141	.iter()	2✔
142	.map(\|v\| {	515✔
143	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
144	csv::Error::UnexpectedValue(	1✔
145	v.clone(),	1✔
146	"Expected quantity while trying to sort by column id".to_string(),	1✔
147	)	1✔
148	})	1✔
149	})	515✔
150	.collect();	2✔
151	let permutation = get_permutation(&col_floats?);	2✔
152	apply_permutation(table, permutation);	1✔
153	Ok(())	1✔
154	}	3✔
155
156	fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	3✔
157	let sort_master_col = table	3✔
158	.columns	3✔
159	.iter()	3✔
160	.find(\|c\| c.header.as_deref().unwrap_or_default() == name)	5✔
161	.ok_or_else(\|\| {	3✔
162	csv::Error::InvalidAccess(format!(	1✔
163	"Requested format sorting by column'{name}' but column not found."	1✔
164	))	1✔
165	})?;	1✔
166	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
167	.rows	2✔
168	.iter()	2✔
169	.map(\|v\| {	515✔
170	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
171	csv::Error::UnexpectedValue(	1✔
172	v.clone(),	1✔
173	"Expected quantity while trying to sort by column name".to_string(),	1✔
174	)	1✔
175	})	1✔
176	})	515✔
177	.collect();	2✔
178	let permutation = get_permutation(&col_floats?);	2✔
179	apply_permutation(table, permutation);	1✔
180	Ok(())	1✔
181	}	3✔
182
183	fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	1✔
184	let pattern = Pattern::new(name).map_err(\|e\| {	1✔
NEW 185	csv::Error::InvalidAccess(format!(	×
NEW 186	"Invalid glob pattern in KeepColumnsByName '{}': {}",	×
NEW 187	name, e	×
NEW 188	))	×
NEW 189	})?;	×
190
191	if let Some(c) = table.columns.iter_mut().find(\|col\| {	2✔
192	col.header.as_deref().unwrap_or_default() == name	2✔
193	\|\| pattern.matches(col.header.as_deref().unwrap_or_default())	1✔
194	}) {	2✔
195	c.delete_contents();	1✔
196	}	1✔
197	Ok(())	1✔
198	}	1✔
199
200	fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {	1✔
201	if let Some(col) = table.columns.get_mut(id) {	1✔
202	col.delete_contents();	1✔
203	}	1✔
204	Ok(())	1✔
205	}	1✔
206
207	fn keep_columns_matching_any_names(	2✔
208	table: &mut Table,	2✔
209	names: &Vec<String>,	2✔
210	) -> Result<(), csv::Error> {	2✔
211	let patterns: Result<Vec<Pattern>, csv::Error> = names	2✔
212	.iter()	2✔
213	.map(\|name\| {	3✔
214	Pattern::new(name).map_err(\|e\| {	3✔
NEW 215	csv::Error::InvalidAccess(format!(	×
NEW 216	"Invalid glob pattern in KeepColumnsByName '{}': {}",	×
NEW 217	name, e	×
NEW 218	))	×
NEW 219	})	×
220	})	3✔
221	.collect();	2✔
222	let patterns = patterns?;	2✔
223
224	table.columns.iter_mut().for_each(\|col\| {	4✔
225	let header = col.header.as_deref().unwrap_or_default();	4✔
226	if !(names.iter().zip(patterns.iter()).any(\|(name, pattern)\| {	5✔
227	// Try exact match first, then glob pattern
228	if header == name {	5✔
229	info!("Keep header: \"{}\" (exact match)", header);	1✔
230	true	1✔
231	} else if pattern.matches(header) {	4✔
232	info!("Keep header: \"{}\" (matches: \"{}\")", header, name);	1✔
233	true	1✔
234	} else {
235	false	3✔
236	}
237	})) {	5✔
238	// info!(	2✔
239	// "Discard column: \"{}\"",	2✔
240	// col.header.as_deref().unwrap_or_default()	2✔
241	// );	2✔
242	col.delete_contents();	2✔
243	}	2✔
244	});	4✔
245	Ok(())	2✔
246	}	2✔
247
248	fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {	155✔
249	debug!("Extracting headers...");	155✔
250	let can_extract = table	155✔
251	.columns	155✔
252	.iter()	155✔
253	.all(\|c\| matches!(c.rows.first(), Some(Value::String(_))));	396✔
254	if !can_extract {	155✔
UNCOV 255	warn!("Cannot extract header for this csv!");	×
UNCOV 256	return Ok(());	×
257	}	155✔
258
259	for col in table.columns.iter_mut() {	396✔
260	let title = col.rows.drain(0..1).next().ok_or_else(\|\| {	396✔
UNCOV 261	csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())	×
UNCOV 262	})?;	×
263	if let Value::String(title) = title {	396✔
264	col.header = Some(title);	396✔
265	}	396✔
266	}
267	Ok(())	155✔
268	}	155✔
269
270	#[cfg(test)]
271	mod tests {
272	use super::*;
273	use crate::csv::{Column, Delimiters, Error};
274	use std::fs::File;
275
276	macro_rules! string_vec {
277	($($x:expr),) => (vec![$($x.to_string()),]);
278	}
279
280	fn setup_table(delimiters: Option<Delimiters>) -> Table {	12✔
281	let delimiters = delimiters.unwrap_or_default();	12✔
282	Table::from_reader(	12✔
283	File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),	12✔
284	&delimiters,	12✔
285	)
286	.unwrap()	12✔
287	}	12✔
288
289	fn setup_table_two(delimiters: Option<Delimiters>) -> Table {	1✔
290	let delimiters = delimiters.unwrap_or_default();	1✔
291	Table::from_reader(	1✔
292	File::open("tests/csv/data/defects_headers.csv").unwrap(),	1✔
293	&delimiters,	1✔
294	)
295	.unwrap()	1✔
296	}	1✔
297
298	#[test]
299	fn test_extract_headers_two() {	1✔
300	let mut table = setup_table_two(None);	1✔
301	extract_headers(&mut table).unwrap();	1✔
302	assert_eq!(	1✔
303	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
304	"Entry"
305	);
306	assert_eq!(	1✔
307	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
308	"Radius"
309	);
310	}	1✔
311
312	#[test]
313	fn test_extract_headers() {	1✔
314	let mut table = setup_table(None);	1✔
315	extract_headers(&mut table).unwrap();	1✔
316	assert_eq!(	1✔
317	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
318	"Deviation [mm]"
319	);
320	assert_eq!(	1✔
321	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
322	"Surface [mm²]"
323	);
324	}	1✔
325
326	#[test]
327	fn test_delete_column_by_id() {	1✔
328	let mut table = setup_table(None);	1✔
329	extract_headers(&mut table).unwrap();	1✔
330	delete_column_number(&mut table, 0).unwrap();	1✔
331	assert_eq!(	1✔
332	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
333	"DELETED"
334	);
335	assert!(table	1✔
336	.columns	1✔
337	.first()	1✔
338	.unwrap()	1✔
339	.rows	1✔
340	.iter()	1✔
341	.all(\|v\| *v == Value::deleted()));	513✔
342	}	1✔
343
344	#[test]
345	fn test_delete_column_by_name() {	1✔
346	let mut table = setup_table(None);	1✔
347	extract_headers(&mut table).unwrap();	1✔
348	delete_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
349	assert_eq!(	1✔
350	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
351	"DELETED"
352	);
353	assert!(table	1✔
354	.columns	1✔
355	.last()	1✔
356	.unwrap()	1✔
357	.rows	1✔
358	.iter()	1✔
359	.all(\|v\| *v == Value::deleted()));	513✔
360	}	1✔
361
362	#[test]
363	fn test_keep_columns_matching_any_names() {	1✔
364	let mut table = setup_table(None);	1✔
365	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
366	extract_headers(&mut table).unwrap();	1✔
367
368	let keep_names = string_vec![	1✔
369	"Deviation [mm]",	1✔
370	"Any name to be kept, which is no table header -> no effect"	1✔
371	];
372	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
373	assert_eq!(	1✔
374	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
375	"Deviation [mm]",
376	"First column was deleted, although it is in the list of names to be kept!"
377	);
378	assert_eq!(	1✔
379	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
380	"DELETED",
381	"Second column was not deleted, although it is not in the list of names to be kept!",
382	);
383	assert!(table	1✔
384	.columns	1✔
385	.last()	1✔
386	.unwrap()	1✔
387	.rows	1✔
388	.iter()	1✔
389	.all(\|v\| *v == Value::deleted()));	513✔
390	}	1✔
391
392	#[test]
393	fn test_keep_columns_matching_glob_patterns() {	1✔
394	let mut table = setup_table(None);	1✔
395	// column headers in test table: "Deviation [mm]", "Surface [mm²]"
396	extract_headers(&mut table).unwrap();	1✔
397
398	let keep_names = string_vec![	1✔
399	"Surface*" // Should match "Surface [mm²]" via glob pattern	1✔
400	];
401	keep_columns_matching_any_names(&mut table, &keep_names).unwrap();	1✔
402	assert_eq!(	1✔
403	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
404	"DELETED",
405	"First column (Deviation) should be deleted as it doesn't match pattern!"
406	);
407	assert_eq!(	1✔
408	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
409	"Surface [mm²]",
410	"Second column (Surface) was deleted, although it matches the glob pattern!",
411	);
412	assert!(table	1✔
413	.columns	1✔
414	.first()	1✔
415	.unwrap()	1✔
416	.rows	1✔
417	.iter()	1✔
418	.all(\|v\| *v == Value::deleted()));	513✔
419	}	1✔
420
421	#[test]
422	fn test_delete_row_by_id() {	1✔
423	let mut table = setup_table(None);	1✔
424	delete_row_by_number(&mut table, 0).unwrap();	1✔
425	assert_eq!(	1✔
426	table	1✔
427	.columns	1✔
428	.first()	1✔
429	.unwrap()	1✔
430	.rows	1✔
431	.first()	1✔
432	.unwrap()	1✔
433	.get_string()	1✔
434	.as_deref()	1✔
435	.unwrap(),	1✔
436	"DELETED"
437	);
438	}	1✔
439
440	#[test]
441	fn test_delete_row_by_regex() {	1✔
442	let mut table = setup_table(None);	1✔
443	delete_row_by_regex(&mut table, "mm").unwrap();	1✔
444	assert_eq!(	1✔
445	table	1✔
446	.columns	1✔
447	.first()	1✔
448	.unwrap()	1✔
449	.rows	1✔
450	.first()	1✔
451	.unwrap()	1✔
452	.get_string()	1✔
453	.as_deref()	1✔
454	.unwrap(),	1✔
455	"DELETED"
456	);
457	}	1✔
458
459	#[test]
460	fn test_sort_by_name() {	1✔
461	let mut table = setup_table(None);	1✔
462	extract_headers(&mut table).unwrap();	1✔
463	sort_by_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
464	let mut peekable_rows = table.rows().peekable();	1✔
465	while let Some(row) = peekable_rows.next() {	514✔
466	if let Some(next_row) = peekable_rows.peek() {	513✔
467	assert!(	512✔
468	row.get(1).unwrap().get_quantity().unwrap().value	512✔
469	>= next_row.get(1).unwrap().get_quantity().unwrap().value	512✔
470	);
471	}	1✔
472	}
473	}	1✔
474
475	#[test]
476	fn test_sort_by_id() {	1✔
477	let mut table = setup_table(None);	1✔
478	extract_headers(&mut table).unwrap();	1✔
479	let column = 1;	1✔
480	sort_by_column_id(&mut table, column).unwrap();	1✔
481	let mut peekable_rows = table.rows().peekable();	1✔
482	while let Some(row) = peekable_rows.next() {	514✔
483	if let Some(next_row) = peekable_rows.peek() {	513✔
484	assert!(	512✔
485	row.get(column).unwrap().get_quantity().unwrap().value	512✔
486	>= next_row.get(column).unwrap().get_quantity().unwrap().value	512✔
487	);
488	}	1✔
489	}
490	}	1✔
491
492	#[test]
493	fn sorting_by_mixed_column_fails() {	1✔
494	let column = Column {	1✔
495	header: Some("Field".to_string()),	1✔
496	rows: vec![	1✔
497	Value::from_str("1.0", &None),	1✔
498	Value::String("String-Value".to_string()),	1✔
499	],	1✔
500	};	1✔
501	let mut table = Table {	1✔
502	columns: vec![column],	1✔
503	};	1✔
504	let order_by_name = sort_by_column_name(&mut table, "Field");	1✔
505	assert!(matches!(	1✔
506	order_by_name.unwrap_err(),	1✔
507	Error::UnexpectedValue(_, _)
508	));
509
510	let order_by_id = sort_by_column_id(&mut table, 0);	1✔
511	assert!(matches!(	1✔
512	order_by_id.unwrap_err(),	1✔
513	Error::UnexpectedValue(_, _)
514	));
515	}	1✔
516
517	#[test]
518	fn non_existing_table_fails() {	1✔
519	let mut table = setup_table(None);	1✔
520	let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");	1✔
521	assert!(matches!(	1✔
522	order_by_name.unwrap_err(),	1✔
523	Error::InvalidAccess(_)
524	));
525
526	let order_by_id = sort_by_column_id(&mut table, 999);	1✔
527	assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));	1✔
528	}	1✔
529
530	#[test]
531	fn test_delete_cell_by_numb() {	1✔
532	let mut table = setup_table(None);	1✔
533	delete_cell_by_number(&mut table, 1, 2).unwrap();	1✔
534
535	assert_eq!(	1✔
536	table	1✔
537	.columns	1✔
538	.get(1)	1✔
539	.unwrap()	1✔
540	.rows	1✔
541	.get(2)	1✔
542	.unwrap()	1✔
543	.get_string()	1✔
544	.as_deref()	1✔
545	.unwrap(),	1✔
546	"DELETED"
547	);
548
549	assert_ne!(	1✔
550	table	1✔
551	.columns	1✔
552	.get(1)	1✔
553	.unwrap()	1✔
554	.rows	1✔
555	.first()	1✔
556	.unwrap()	1✔
557	.get_string()	1✔
558	.as_deref()	1✔
559	.unwrap(),	1✔
560	"DELETED"
561	);
562
563	assert_eq!(	1✔
564	table	1✔
565	.columns	1✔
566	.first()	1✔
567	.unwrap()	1✔
568	.rows	1✔
569	.get(1)	1✔
570	.unwrap()	1✔
571	.get_string(),	1✔
572	None
573	);
574	}	1✔
575
576	#[test]
577	fn test_delete_cell_by_name() {	1✔
578	let mut table = setup_table(None);	1✔
579	extract_headers(&mut table).unwrap();	1✔
580	delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap();	1✔
581
582	assert_eq!(	1✔
583	table	1✔
584	.columns	1✔
585	.get(1)	1✔
586	.unwrap()	1✔
587	.rows	1✔
588	.get(1)	1✔
589	.unwrap()	1✔
590	.get_string()	1✔
591	.as_deref()	1✔
592	.unwrap(),	1✔
593	"DELETED"
594	);
595
596	assert_eq!(	1✔
597	table	1✔
598	.columns	1✔
599	.get(1)	1✔
600	.unwrap()	1✔
601	.rows	1✔
602	.get(3)	1✔
603	.unwrap()	1✔
604	.get_string(),	1✔
605	None
606	);
607
608	assert_eq!(	1✔
609	table	1✔
610	.columns	1✔
611	.get(0)	1✔
612	.unwrap()	1✔
613	.rows	1✔
614	.get(1)	1✔
615	.unwrap()	1✔
616	.get_string(),	1✔
617	None
618	);
619	}	1✔
620	}

VolumeGraphics / havocompare / 21443895732

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous