cbb3d470341b379022395dd011fd0a6f21bf72d1

Build # cbb3d470341b379022395dd011fd0a6f21bf72d1

Build Type

push

github

Committed by GitHub

Commit Message

Merge pull request #24 from VolumeGraphics/fix-toml-version-0.2.2

Run Details

2367 of 2572 relevant lines covered (92.03%)

2279.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.2

/src/csv/preprocessing.rs

use crate::csv;
use crate::csv::value::Value;
use crate::csv::Table;
use schemars_derive::JsonSchema;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering::Equal;
use tracing::{debug, warn};

#[derive(JsonSchema, Deserialize, Serialize, Debug)]
/// Preprocessor options
pub enum Preprocessor {
    /// Try to extract the headers from the first row - fallible if first row contains a number
    ExtractHeaders,
    /// Replace all fields in column by number by a deleted marker
    DeleteColumnByNumber(usize),
    /// Replace all fields in column by name by a deleted marker
    DeleteColumnByName(String),
    /// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
    SortByColumnName(String),
    /// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
    SortByColumnNumber(usize),
    /// Replace all fields in row with given number by a deleted marker
    DeleteRowByNumber(usize),
    /// Replace all fields in row  where at least a single field matches regex by a deleted marker
    DeleteRowByRegex(String),
}

impl Preprocessor {
    pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {
        match self {
            Preprocessor::ExtractHeaders => extract_headers(table),
            Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),
            Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),
            Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),
            Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),
            Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),
            Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),
        }
    }
}

fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {
    let regex = regex::Regex::new(regex)?;
    table
        .rows_mut()
        .filter(|row| row.iter().any(|v| regex.is_match(v.to_string().as_str())))
        .for_each(|mut row| row.iter_mut().for_each(|v| **v = Value::deleted()));
    Ok(())
}

fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(mut v) = table.rows_mut().nth(id) {
        v.iter_mut().for_each(|v| **v = Value::deleted())
    }
    Ok(())
}

fn get_permutation(rows_to_sort_by: &Vec<f32>) -> permutation::Permutation {
    permutation::sort_by(rows_to_sort_by, |a, b| b.partial_cmp(a).unwrap_or(Equal))
}

fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {
    table.columns.iter_mut().for_each(|c| {
        permutation.apply_slice_in_place(&mut c.rows);
    });
}

fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    let sort_master_col = table.columns.get(id).ok_or_else(|| {
        csv::Error::InvalidAccess(format!(
            "Column number sorting by id {} requested but column not found.",
            id
        ))
    })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column id".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    let sort_master_col = table
        .columns
        .iter()
        .find(|c| c.header.as_deref().unwrap_or_default() == name)
        .ok_or_else(|| {
            csv::Error::InvalidAccess(format!(
                "Requested format sorting by column'{}' but column not found.",
                name
            ))
        })?;
    let col_floats: Result<Vec<_>, csv::Error> = sort_master_col
        .rows
        .iter()
        .map(|v| {
            v.get_quantity().map(|q| q.value).ok_or_else(|| {
                csv::Error::UnexpectedValue(
                    v.clone(),
                    "Expected quantity while trying to sort by column name".to_string(),
                )
            })
        })
        .collect();
    let permutation = get_permutation(&col_floats?);
    apply_permutation(table, permutation);
    Ok(())
}

fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
    if let Some(c) = table
        .columns
        .iter_mut()
        .find(|col| col.header.as_deref().unwrap_or_default() == name)
    {
        c.delete_contents();
    }
    Ok(())
}

fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
    if let Some(col) = table.columns.get_mut(id) {
        col.delete_contents();
    }
    Ok(())
}

fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {
    debug!("Extracting headers...");
    let can_extract = table
        .columns
        .iter()
        .all(|c| matches!(c.rows.first(), Some(Value::String(_))));
    if !can_extract {
        warn!("Cannot extract header for this csv!");
        return Ok(());
    }

    for col in table.columns.iter_mut() {
        let title = col.rows.drain(0..1).next().ok_or_else(|| {
            csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())
        })?;
        if let Value::String(title) = title {
            col.header = Some(title);
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::csv::{Column, Delimiters, Error};
    use std::fs::File;

    fn setup_table(delimiters: Option<Delimiters>) -> Table {
        let delimiters = delimiters.unwrap_or_default();
        Table::from_reader(
            File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),
            &delimiters,
        )
        .unwrap()
    }

    #[test]
    fn test_extract_headers() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "Deviation [mm]"
        );
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "Surface [mm²]"
        );
    }

    #[test]
    fn test_delete_column_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_number(&mut table, 0).unwrap();
        assert_eq!(
            table.columns.first().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .first()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == csv::Value::deleted()));
    }

    #[test]
    fn test_delete_column_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        delete_column_name(&mut table, "Surface [mm²]").unwrap();
        assert_eq!(
            table.columns.last().unwrap().header.as_deref().unwrap(),
            "DELETED"
        );
        assert!(table
            .columns
            .last()
            .unwrap()
            .rows
            .iter()
            .all(|v| *v == csv::Value::deleted()));
    }

    #[test]
    fn test_delete_row_by_id() {
        let mut table = setup_table(None);
        delete_row_by_number(&mut table, 0).unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_delete_row_by_regex() {
        let mut table = setup_table(None);
        delete_row_by_regex(&mut table, "mm").unwrap();
        assert_eq!(
            table
                .columns
                .first()
                .unwrap()
                .rows
                .first()
                .unwrap()
                .get_string()
                .as_deref()
                .unwrap(),
            "DELETED"
        );
    }

    #[test]
    fn test_sort_by_name() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        sort_by_column_name(&mut table, "Surface [mm²]").unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(1).unwrap().get_quantity().unwrap().value
                        >= next_row.get(1).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn test_sort_by_id() {
        let mut table = setup_table(None);
        extract_headers(&mut table).unwrap();
        let column = 1;
        sort_by_column_id(&mut table, column).unwrap();
        let mut peekable_rows = table.rows().peekable();
        while let Some(row) = peekable_rows.next() {
            if let Some(next_row) = peekable_rows.peek() {
                assert!(
                    row.get(column).unwrap().get_quantity().unwrap().value
                        >= next_row.get(column).unwrap().get_quantity().unwrap().value
                );
            }
        }
    }

    #[test]
    fn sorting_by_mixed_column_fails() {
        let column = Column {
            header: Some("Field".to_string()),
            rows: vec![
                Value::from_str("1.0", &None),
                Value::String("String-Value".to_string()),
            ],
        };
        let mut table = Table {
            columns: vec![column],
        };
        let order_by_name = sort_by_column_name(&mut table, "Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));

        let order_by_id = sort_by_column_id(&mut table, 0);
        assert!(matches!(
            order_by_id.unwrap_err(),
            Error::UnexpectedValue(_, _)
        ));
    }

    #[test]
    fn non_existing_table_fails() {
        let mut table = setup_table(None);
        let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");
        assert!(matches!(
            order_by_name.unwrap_err(),
            Error::InvalidAccess(_)
        ));

        let order_by_id = sort_by_column_id(&mut table, 999);
        assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));
    }
}

1	use crate::csv;
2	use crate::csv::value::Value;
3	use crate::csv::Table;
4	use schemars_derive::JsonSchema;
5	use serde::{Deserialize, Serialize};
6	use std::cmp::Ordering::Equal;
7	use tracing::{debug, warn};
8
9	#[derive(JsonSchema, Deserialize, Serialize, Debug)]	2✔
10	/// Preprocessor options
11	pub enum Preprocessor {
12	/// Try to extract the headers from the first row - fallible if first row contains a number
13	ExtractHeaders,
14	/// Replace all fields in column by number by a deleted marker
15	DeleteColumnByNumber(usize),
16	/// Replace all fields in column by name by a deleted marker
17	DeleteColumnByName(String),
18	/// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there
19	SortByColumnName(String),
20	/// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds.
21	SortByColumnNumber(usize),
22	/// Replace all fields in row with given number by a deleted marker
23	DeleteRowByNumber(usize),
24	/// Replace all fields in row where at least a single field matches regex by a deleted marker
25	DeleteRowByRegex(String),
26	}
27
28	impl Preprocessor {
29	pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> {	72✔
30	match self {	72✔
31	Preprocessor::ExtractHeaders => extract_headers(table),	72✔
32	Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id),	×
33	Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()),	×
34	Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()),	×
35	Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id),	×
36	Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id),	×
37	Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex),	×
38	}
39	}	72✔
40	}
41
42	fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> {	1✔
43	let regex = regex::Regex::new(regex)?;	1✔
44	table	1✔
45	.rows_mut()	1✔
46	.filter(\|row\| row.iter().any(\|v\| regex.is_match(v.to_string().as_str())))	1,027✔
47	.for_each(\|mut row\| row.iter_mut().for_each(\|v\| **v = Value::deleted()));	2✔
48	Ok(())	1✔
49	}	1✔
50
51	fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
52	if let Some(mut v) = table.rows_mut().nth(id) {	1✔
53	v.iter_mut().for_each(\|v\| **v = Value::deleted())	2✔
54	}	×
55	Ok(())	1✔
56	}	1✔
57
58	fn get_permutation(rows_to_sort_by: &Vec<f32>) -> permutation::Permutation {	2✔
59	permutation::sort_by(rows_to_sort_by, \|a, b\| b.partial_cmp(a).unwrap_or(Equal))	4,228✔
60	}	2✔
61
62	fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) {	2✔
63	table.columns.iter_mut().for_each(\|c\| {	4✔
64	permutation.apply_slice_in_place(&mut c.rows);	4✔
65	});	4✔
66	}	2✔
67
68	fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> {	3✔
69	let sort_master_col = table.columns.get(id).ok_or_else(\|\| {	3✔
70	csv::Error::InvalidAccess(format!(	1✔
71	"Column number sorting by id {} requested but column not found.",	1✔
72	id	1✔
73	))	1✔
74	})?;	3✔
75	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
76	.rows	2✔
77	.iter()	2✔
78	.map(\|v\| {	515✔
79	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
80	csv::Error::UnexpectedValue(	1✔
81	v.clone(),	1✔
82	"Expected quantity while trying to sort by column id".to_string(),	1✔
83	)	1✔
84	})	515✔
85	})	515✔
86	.collect();	2✔
87	let permutation = get_permutation(&col_floats?);	2✔
88	apply_permutation(table, permutation);	1✔
89	Ok(())	1✔
90	}	3✔
91
92	fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {	3✔
93	let sort_master_col = table	3✔
94	.columns	3✔
95	.iter()	3✔
96	.find(\|c\| c.header.as_deref().unwrap_or_default() == name)	5✔
97	.ok_or_else(\|\| {	3✔
98	csv::Error::InvalidAccess(format!(	1✔
99	"Requested format sorting by column'{}' but column not found.",	1✔
100	name	1✔
101	))	1✔
102	})?;	3✔
103	let col_floats: Result<Vec<_>, csv::Error> = sort_master_col	2✔
104	.rows	2✔
105	.iter()	2✔
106	.map(\|v\| {	515✔
107	v.get_quantity().map(\|q\| q.value).ok_or_else(\|\| {	515✔
108	csv::Error::UnexpectedValue(	1✔
109	v.clone(),	1✔
110	"Expected quantity while trying to sort by column name".to_string(),	1✔
111	)	1✔
112	})	515✔
113	})	515✔
114	.collect();	2✔
115	let permutation = get_permutation(&col_floats?);	2✔
116	apply_permutation(table, permutation);	1✔
117	Ok(())	1✔
118	}	3✔
119
120	fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> {
121	if let Some(c) = table	1✔
122	.columns	1✔
123	.iter_mut()	1✔
124	.find(\|col\| col.header.as_deref().unwrap_or_default() == name)	2✔
125	{	1✔
126	c.delete_contents();	1✔
127	}	1✔
128	Ok(())	1✔
129	}	1✔
130
131	fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> {
132	if let Some(col) = table.columns.get_mut(id) {	1✔
133	col.delete_contents();	1✔
134	}	1✔
135	Ok(())	1✔
136	}	1✔
137
138	fn extract_headers(table: &mut Table) -> Result<(), csv::Error> {	77✔
139	debug!("Extracting headers...");	77✔
140	let can_extract = table	77✔
141	.columns	77✔
142	.iter()	77✔
143	.all(\|c\| matches!(c.rows.first(), Some(Value::String(_))));	186✔
144	if !can_extract {	77✔
145	warn!("Cannot extract header for this csv!");	×
146	return Ok(());	×
147	}	77✔
148
149	for col in table.columns.iter_mut() {	186✔
150	let title = col.rows.drain(0..1).next().ok_or_else(\|\| {	186✔
151	csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string())	×
152	})?;	186✔
153	if let Value::String(title) = title {	186✔
154	col.header = Some(title);	186✔
155	}	186✔
156	}
157	Ok(())	77✔
158	}	77✔
159
160	#[cfg(test)]
161	mod tests {
162	use super::*;
163	use crate::csv::{Column, Delimiters, Error};
164	use std::fs::File;
165
166	fn setup_table(delimiters: Option<Delimiters>) -> Table {	8✔
167	let delimiters = delimiters.unwrap_or_default();	8✔
168	Table::from_reader(	8✔
169	File::open("tests/csv/data/DeviationHistogram.csv").unwrap(),	8✔
170	&delimiters,	8✔
171	)	8✔
172	.unwrap()	8✔
173	}	8✔
174
175	#[test]	1✔
176	fn test_extract_headers() {	1✔
177	let mut table = setup_table(None);	1✔
178	extract_headers(&mut table).unwrap();	1✔
179	assert_eq!(	1✔
180	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
181	"Deviation [mm]"	1✔
182	);	1✔
183	assert_eq!(	1✔
184	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
185	"Surface [mm²]"	1✔
186	);	1✔
187	}	1✔
188
189	#[test]	1✔
190	fn test_delete_column_by_id() {	1✔
191	let mut table = setup_table(None);	1✔
192	extract_headers(&mut table).unwrap();	1✔
193	delete_column_number(&mut table, 0).unwrap();	1✔
194	assert_eq!(	1✔
195	table.columns.first().unwrap().header.as_deref().unwrap(),	1✔
196	"DELETED"	1✔
197	);	1✔
198	assert!(table	1✔
199	.columns	1✔
200	.first()	1✔
201	.unwrap()	1✔
202	.rows	1✔
203	.iter()	1✔
204	.all(\|v\| *v == csv::Value::deleted()));	513✔
205	}	1✔
206
207	#[test]	1✔
208	fn test_delete_column_by_name() {	1✔
209	let mut table = setup_table(None);	1✔
210	extract_headers(&mut table).unwrap();	1✔
211	delete_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
212	assert_eq!(	1✔
213	table.columns.last().unwrap().header.as_deref().unwrap(),	1✔
214	"DELETED"	1✔
215	);	1✔
216	assert!(table	1✔
217	.columns	1✔
218	.last()	1✔
219	.unwrap()	1✔
220	.rows	1✔
221	.iter()	1✔
222	.all(\|v\| *v == csv::Value::deleted()));	513✔
223	}	1✔
224
225	#[test]	1✔
226	fn test_delete_row_by_id() {	1✔
227	let mut table = setup_table(None);	1✔
228	delete_row_by_number(&mut table, 0).unwrap();	1✔
229	assert_eq!(	1✔
230	table	1✔
231	.columns	1✔
232	.first()	1✔
233	.unwrap()	1✔
234	.rows	1✔
235	.first()	1✔
236	.unwrap()	1✔
237	.get_string()	1✔
238	.as_deref()	1✔
239	.unwrap(),	1✔
240	"DELETED"	1✔
241	);	1✔
242	}	1✔
243
244	#[test]	1✔
245	fn test_delete_row_by_regex() {	1✔
246	let mut table = setup_table(None);	1✔
247	delete_row_by_regex(&mut table, "mm").unwrap();	1✔
248	assert_eq!(	1✔
249	table	1✔
250	.columns	1✔
251	.first()	1✔
252	.unwrap()	1✔
253	.rows	1✔
254	.first()	1✔
255	.unwrap()	1✔
256	.get_string()	1✔
257	.as_deref()	1✔
258	.unwrap(),	1✔
259	"DELETED"	1✔
260	);	1✔
261	}	1✔
262
263	#[test]	1✔
264	fn test_sort_by_name() {	1✔
265	let mut table = setup_table(None);	1✔
266	extract_headers(&mut table).unwrap();	1✔
267	sort_by_column_name(&mut table, "Surface [mm²]").unwrap();	1✔
268	let mut peekable_rows = table.rows().peekable();	1✔
269	while let Some(row) = peekable_rows.next() {	514✔
270	if let Some(next_row) = peekable_rows.peek() {	513✔
271	assert!(	512✔
272	row.get(1).unwrap().get_quantity().unwrap().value	512✔
273	>= next_row.get(1).unwrap().get_quantity().unwrap().value	512✔
274	);	512✔
275	}	1✔
276	}
277	}	1✔
278
279	#[test]	1✔
280	fn test_sort_by_id() {	1✔
281	let mut table = setup_table(None);	1✔
282	extract_headers(&mut table).unwrap();	1✔
283	let column = 1;	1✔
284	sort_by_column_id(&mut table, column).unwrap();	1✔
285	let mut peekable_rows = table.rows().peekable();	1✔
286	while let Some(row) = peekable_rows.next() {	514✔
287	if let Some(next_row) = peekable_rows.peek() {	513✔
288	assert!(	512✔
289	row.get(column).unwrap().get_quantity().unwrap().value	512✔
290	>= next_row.get(column).unwrap().get_quantity().unwrap().value	512✔
291	);	512✔
292	}	1✔
293	}
294	}	1✔
295
296	#[test]	1✔
297	fn sorting_by_mixed_column_fails() {	1✔
298	let column = Column {	1✔
299	header: Some("Field".to_string()),	1✔
300	rows: vec![	1✔
301	Value::from_str("1.0", &None),	1✔
302	Value::String("String-Value".to_string()),	1✔
303	],	1✔
304	};	1✔
305	let mut table = Table {	1✔
306	columns: vec![column],	1✔
307	};	1✔
308	let order_by_name = sort_by_column_name(&mut table, "Field");	1✔
309	assert!(matches!(	1✔
310	order_by_name.unwrap_err(),	1✔
311	Error::UnexpectedValue(_, _)
312	));
313
314	let order_by_id = sort_by_column_id(&mut table, 0);	1✔
315	assert!(matches!(	1✔
316	order_by_id.unwrap_err(),	1✔
317	Error::UnexpectedValue(_, _)
318	));
319	}	1✔
320
321	#[test]	1✔
322	fn non_existing_table_fails() {	1✔
323	let mut table = setup_table(None);	1✔
324	let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field");	1✔
325	assert!(matches!(	1✔
326	order_by_name.unwrap_err(),	1✔
327	Error::InvalidAccess(_)
328	));
329
330	let order_by_id = sort_by_column_id(&mut table, 999);	1✔
331	assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_)));	1✔
332	}	1✔
333	}

VolumeGraphics / havocompare / cbb3d470341b379022395dd011fd0a6f21bf72d1

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous