18363067970

Committed 09 Oct 2025 01:51AM UTC coverage: 81.743% (+15.5%) from 66.205%

Build # 18363067970

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #39 from juarezr/feat/enhancements

Enhancements and Improvements:

- Added --archive-compression flag to the backup command with support for three compression methods
- Comprehensive Makefile.toml with organized task categories
- Coverage reporting with HTML output generation
- VS Code configuration with recommended extensions

Run Details

343 of 346 new or added lines in 7 files covered. (99.13%)

4 existing lines in 3 files now uncovered.

1791 of 2191 relevant lines covered (81.74%)

3.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.33

/src/steps.rs

use super::{
    args::{Backup, IterateMode},
    fails::{BoxedResult, throw},
    helpers::{BRACKETS, COMMA, EMPTY_STR, EMPTY_STRING},
    helpers::{IntegerHelpers, StringHelpers, replace_solr_date, solr_query},
    models::{SolrCore, Step},
};
use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, Utc};
use log::debug;
use std::collections::HashSet;
use std::iter::FromIterator;

// region Data Structures

#[derive(Debug)]
pub(crate) struct Slices<T> {
    pub curr: T,
    pub end: T,
    pub increment: u64,
    pub mode: IterateMode,
}

#[derive(Debug)]
pub(crate) struct SliceItem {
    pub begin: String,
    pub end: String,
}

#[derive(Debug, Clone)]
pub(crate) struct Requests {
    pub curr: u64,
    pub limit: u64,
    pub num_docs: u64,
    pub url: String,
}

// endregion

// region Iterators

impl Slices<String> {
    pub(crate) fn get_iterator(&self) -> Box<dyn Iterator<Item = SliceItem>> {
        if self.curr.is_empty() {
            return Box::new(Self::get_slice_of(1, 1));
        }
        let res: Box<dyn Iterator<Item = SliceItem>> = match self.mode {
            IterateMode::None => Box::new(Self::get_slice_of(1, 1)),
            IterateMode::Range => Box::new(self.get_range_slices().unwrap()),
            _ => Box::new(self.get_period_slices().unwrap()),
        };
        res
    }

    pub(crate) fn estimate_steps(&self) -> BoxedResult<u64> {
        if self.curr.is_empty() {
            return Ok(1);
        }
        let num: u64 = match self.mode {
            IterateMode::None => 1,
            IterateMode::Range => self.get_range_slices()?.len(),
            _ => self.get_period_slices()?.len(),
        };
        let rem = num % self.increment;
        if rem > 0 { Ok(num + 1) } else { Ok(num) }
    }

    fn get_slice_of(num: u64, incr: u64) -> Slices<u64> {
        Slices::<u64> { curr: 0, end: num, mode: IterateMode::Range, increment: incr }
    }

    fn get_range_slices(&self) -> BoxedResult<Slices<u64>> {
        let v1 = Self::parse_between_number(self.curr.as_str())?;
        let v2 = Self::parse_between_number(self.end.as_str())?;
        Ok(Slices::<u64> { curr: v1, end: v2, increment: self.increment, mode: IterateMode::Range })
    }

    fn get_period_slices(&self) -> BoxedResult<Slices<NaiveDateTime>> {
        let v1 = Self::parse_between_date(self.curr.as_str())?;
        let v2 = Self::parse_between_date(self.end.as_str())?;
        Ok(Slices::<NaiveDateTime> {
            curr: v1,
            end: v2,
            increment: self.increment,
            mode: self.mode,
        })
    }

    fn parse_between_number(value: &str) -> BoxedResult<u64> {
        let parsed = value.parse::<u64>();
        match parsed {
            Err(_) => throws!("Wrong value for number: {}", value),
            Ok(quantity) => Ok(quantity),
        }
    }

    fn parse_between_date(value: &str) -> BoxedResult<NaiveDateTime> {
        if value.contains('T') {
            let time = value.parse::<NaiveDateTime>();
            match time {
                Err(_) => throws!("Wrong value for datetime: {}", value),
                Ok(quantity) => Ok(quantity),
            }
        } else {
            let date = value.parse::<NaiveDate>();
            match date {
                Err(_) => throw(format!("Wrong value for date: '{}'", value)),
                Ok(quantity) => quantity
                    .and_hms_opt(0, 0, 0)
                    .ok_or_else(should_fail!("Wrong value for date: '{}'", value)),
            }
        }
    }
}

impl Slices<NaiveDateTime> {
    fn get_interval(&self, less: i64) -> Duration {
        let plus = self.increment.to_i64();
        let dur = match self.mode {
            IterateMode::Minute => Duration::minutes(plus),
            IterateMode::Hour => Duration::hours(plus),
            IterateMode::Day => Duration::days(plus),
            _ => Duration::days(365),
        };
        dur - Duration::seconds(less)
    }

    fn len(&self) -> u64 {
        let dur = self.end - self.curr;
        let (diff, prev, div) = match self.mode {
            IterateMode::Minute => (dur.num_minutes(), dur.num_seconds(), 60i64),
            IterateMode::Hour => (dur.num_hours(), dur.num_minutes(), 60i64),
            IterateMode::Day => (dur.num_days(), dur.num_hours(), 24i64),
            _ => (1i64, 1i64, 1i64),
        };
        if diff < 0 {
            0
        } else {
            let rem = prev % div;
            let res = if rem == 0 { diff } else { diff + 1 };
            res.to_u64()
        }
    }
}

impl Slices<u64> {
    fn len(&self) -> u64 {
        self.end - self.curr + 1
    }
}

impl Iterator for Slices<u64> {
    type Item = SliceItem;

    fn next(&mut self) -> Option<Self::Item> {
        if self.end > self.curr {
            let next = self.curr + self.increment;
            let last = next - 1;
            let res = SliceItem { begin: self.curr.to_string(), end: last.to_string() };
            self.curr = next;
            Some(res)
        } else {
            None
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let num_steps = self.len();
        if num_steps == 0 { (0, None) } else { (0, Some(num_steps.to_usize())) }
    }
}

impl Iterator for Slices<NaiveDateTime> {
    type Item = SliceItem;

    fn next(&mut self) -> Option<Self::Item> {
        if self.end > self.curr {
            let last = self.curr + self.get_interval(1);
            let part = if last < self.end { last } else { self.end };
            let res = SliceItem { begin: format_solr_time(self.curr), end: format_solr_time(part) };
            let next = self.get_interval(0);
            self.curr += next;
            Some(res)
        } else {
            None
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let num_steps = self.len();
        if num_steps == 0 { (0, None) } else { (0, Some(num_steps.to_usize())) }
    }
}

impl SliceItem {
    pub(crate) fn filter(&self, step: Step) -> Step {
        if self.begin.is_empty() {
            step
        } else {
            let query =
                replace_solr_vars(step.url.as_str(), self.begin.as_str(), self.end.as_str());
            Step { url: query, curr: step.curr }
        }
    }
}

impl Requests {
    pub(crate) fn len(&self) -> u64 {
        let res = self.limit / self.num_docs;
        if self.limit % self.num_docs == 0 { res } else { res + 1 }
    }
}

impl Iterator for Requests {
    type Item = Step;

    fn next(&mut self) -> Option<Step> {
        if self.limit > self.curr {
            let remaining = self.limit - self.curr;
            let rows = self.num_docs.min(remaining);
            let query = format!("{}&start={}&rows={}", self.url, self.curr, rows);
            let res = Step { url: query, curr: self.curr };
            self.curr += self.num_docs;
            Some(res)
        } else {
            None
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let num_steps = self.len();
        if num_steps == 0 {
            (0, None)
        } else {
            let max: usize = num_steps.to_usize();
            (0, Some(max))
        }
    }
}

fn replace_solr_vars(query: &str, begin: &str, end: &str) -> String {
    let query2 = replace_solr_date(query, "{begin}", begin);
    replace_solr_date(&query2, "{end}", end)
}

fn format_solr_time(date_time: NaiveDateTime) -> String {
    date_time.format("%Y-%m-%dT%H:%M:%SZ").to_string()
}

// endregion

// region Solr requests

impl Backup {
    pub(crate) fn get_archive_pattern(&self, num_found: u64) -> String {
        let prefix = match &self.archive_prefix {
            Some(text) => text.to_string(),
            None => {
                let now: DateTime<Utc> = Utc::now();
                let time = now.format("%Y%m%d_%H%M");
                format!("{}_at_{}", &self.options.core, &time)
            }
        };
        let ext = self.archive_compression.get_ext();
        format!("{}_docs_{}_seq_{}.{}", prefix, num_found, BRACKETS, ext)
    }

    pub(crate) fn estimate_docs_quantity(
        &self, schema: &SolrCore, slices: &Slices<String>,
    ) -> BoxedResult<u64> {
        let end_limit = self.get_docs_to_retrieve(schema);
        let num_retrieving = end_limit - self.skip;

        let slice_count = slices.estimate_steps()?;
        Ok(num_retrieving * slice_count)
    }

    pub(crate) fn get_docs_to_retrieve(&self, schema: &SolrCore) -> u64 {
        schema.num_found.min(self.limit.unwrap_or(u64::MAX))
    }

    pub(crate) fn get_steps(&self, schema: &SolrCore) -> Requests {
        let include_hash: HashSet<String> = HashSet::from_iter(schema.fields.clone());
        let exclude_hash: HashSet<String> = HashSet::from_iter(self.exclude.clone());
        let diff: Vec<String> = include_hash.difference(&exclude_hash).map(String::from).collect();

        debug!("Include fields {:?}", include_hash);
        debug!("Exclude fields {:?}", exclude_hash);
        debug!("Actual fields  {:?}", diff);

        let fl = self.get_query_fields(diff);
        let query = self.get_query_url(&fl, true);
        let end_limit = self.get_docs_to_retrieve(schema);
        Requests { curr: self.skip, limit: end_limit, num_docs: self.num_docs, url: query }
    }

    pub(crate) fn get_query_fields(&self, core_fields: Vec<String>) -> String {
        if core_fields.is_empty() {
            EMPTY_STRING
        } else {
            let all = core_fields.join(COMMA);
            "&fl=".append(&all)
        }
    }

    pub(crate) fn get_query_for_diagnostics(&self) -> String {
        let url = self.get_query_url(EMPTY_STR, false);
        format!("{}&start=0&rows=1", url)
    }

    pub(crate) fn replace_vars(&self, query: &str, raw: bool) -> String {
        if raw || self.iterate_between.is_empty() {
            query.to_string()
        } else {
            let (begin, end) = self.get_between();
            replace_solr_vars(query, begin, end)
        }
    }

    pub(crate) fn get_query_url(&self, selected: &str, raw: bool) -> String {
        let qparam = self.query.as_deref().unwrap_or("*:*");
        let qfixed = self.replace_vars(qparam, raw);
        let filterq = solr_query(&qfixed);
        let fqparam = self.fq.as_deref().unwrap_or("*:*");
        let filterfq = solr_query(fqparam);

        let sort: String = if self.order.is_empty() {
            EMPTY_STRING
        } else {
            let all: Vec<String> = self.order.iter().map(|field| field.to_string()).collect();
            let joined = all.join(COMMA);
            "&sort=".append(&joined)
        };
        let parts = [
            self.options.url.with_suffix("/"),
            self.options.core.clone(),
            "/select?wt=json&indent=off&omitHeader=true".to_string(),
            format!("&q={}", filterq),
            format!("&fq={}", filterfq),
            sort,
            self.transfer.get_param("&"),
            selected.to_string(),
        ];
        parts.concat()
    }

    pub(crate) fn get_slices(&self) -> Slices<String> {
        let (begin, end) = self.get_between();
        Slices::<String> {
            curr: begin.to_string(),
            end: end.to_string(),
            increment: self.iterate_step,
            mode: self.iterate_by,
        }
    }

    fn get_between(&self) -> (&str, &str) {
        if self.iterate_between.is_empty() {
            (EMPTY_STR, EMPTY_STR)
        } else {
            (self.iterate_between[0].as_str(), self.iterate_between[1].as_str())
        }
    }
}

// endregion

#[cfg(test)]
mod tests {
    // region mockup

    use crate::{
        args::{Backup, Cli, Commands, IterateMode, shared::TEST_SELECT_FIELDS},
        fails::{BoxedResult, raise},
        helpers::{COMMA, EMPTY_STR},
        steps::{Slices, SolrCore},
    };
    use pretty_assertions::assert_eq;

    impl Commands {
        pub(crate) fn get(&self) -> BoxedResult<&Backup> {
            match &self {
                Self::Backup(gets) => Ok(&gets),
                _ => raise("command must be 'backup' !"),
            }
        }
    }

    impl SolrCore {
        pub(crate) fn mockup() -> Self {
            SolrCore { num_found: 100, fields: vec![TEST_SELECT_FIELDS.split(COMMA).collect()] }
        }
    }

    // endregion

    // region iterators

    #[test]
    fn check_iterator_for_params_get() {
        let parsed = Cli::mockup_args_backup();
        let gets = parsed.get().unwrap();
        let core_info = SolrCore::mockup();
        let query = gets.get_query_url(EMPTY_STR, true);

        let mut i = 0;
        for step in gets.get_steps(&core_info) {
            let url = step.url;
            assert_eq!(url.is_empty(), false);
            assert_eq!(url.starts_with(&query), true);
            i += 1;
        }
        assert_eq!(i, 8);
    }

    #[test]
    fn check_iterator_for_slices_u64() {
        let slices = Slices::<String>::get_slice_of(16, 2);
        for step in slices {
            assert!(step.begin < step.end, "# {} -> {}", step.begin, step.end)
        }
    }

    #[test]
    fn check_iterator_for_slices_datetime() {
        let src = Slices::<String> {
            curr: "2020-04-01".to_string(),
            end: "2020-04-03T11:12:13".to_string(),
            increment: 1,
            mode: IterateMode::Day,
        };

        let slices = src.get_period_slices();
        assert!(slices.is_ok());

        if let Ok(seq) = slices {
            for step in seq {
                assert!(step.begin < step.end, "# {} -> {}", step.begin, step.end)
            }
        }
    }

    // endregion
}

1	use super::{
2	args::{Backup, IterateMode},
3	fails::{BoxedResult, throw},
4	helpers::{BRACKETS, COMMA, EMPTY_STR, EMPTY_STRING},
5	helpers::{IntegerHelpers, StringHelpers, replace_solr_date, solr_query},
6	models::{SolrCore, Step},
7	};
8	use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, Utc};
9	use log::debug;
10	use std::collections::HashSet;
11	use std::iter::FromIterator;
12
13	// region Data Structures
14
15	#[derive(Debug)]
16	pub(crate) struct Slices<T> {
17	pub curr: T,
18	pub end: T,
19	pub increment: u64,
20	pub mode: IterateMode,
21	}
22
23	#[derive(Debug)]
24	pub(crate) struct SliceItem {
25	pub begin: String,
26	pub end: String,
27	}
28
29	#[derive(Debug, Clone)]
30	pub(crate) struct Requests {
31	pub curr: u64,
32	pub limit: u64,
33	pub num_docs: u64,
34	pub url: String,
35	}
36
37	// endregion
38
39	// region Iterators
40
41	impl Slices<String> {
42	pub(crate) fn get_iterator(&self) -> Box<dyn Iterator<Item = SliceItem>> {	2✔
43	if self.curr.is_empty() {	2✔
44	return Box::new(Self::get_slice_of(1, 1));	2✔
45	}	×
46	let res: Box<dyn Iterator<Item = SliceItem>> = match self.mode {	×
47	IterateMode::None => Box::new(Self::get_slice_of(1, 1)),	×
48	IterateMode::Range => Box::new(self.get_range_slices().unwrap()),	×
49	_ => Box::new(self.get_period_slices().unwrap()),	×
50	};
51	res	×
52	}	2✔
53
54	pub(crate) fn estimate_steps(&self) -> BoxedResult<u64> {	2✔
55	if self.curr.is_empty() {	2✔
56	return Ok(1);	2✔
57	}	×
58	let num: u64 = match self.mode {	×
59	IterateMode::None => 1,	×
60	IterateMode::Range => self.get_range_slices()?.len(),	×
61	_ => self.get_period_slices()?.len(),	×
62	};
63	let rem = num % self.increment;	×
64	if rem > 0 { Ok(num + 1) } else { Ok(num) }	×
65	}	2✔
66
67	fn get_slice_of(num: u64, incr: u64) -> Slices<u64> {	3✔
68	Slices::<u64> { curr: 0, end: num, mode: IterateMode::Range, increment: incr }	3✔
69	}	3✔
70
71	fn get_range_slices(&self) -> BoxedResult<Slices<u64>> {	×
72	let v1 = Self::parse_between_number(self.curr.as_str())?;	×
73	let v2 = Self::parse_between_number(self.end.as_str())?;	×
74	Ok(Slices::<u64> { curr: v1, end: v2, increment: self.increment, mode: IterateMode::Range })	×
75	}	×
76
77	fn get_period_slices(&self) -> BoxedResult<Slices<NaiveDateTime>> {	1✔
78	let v1 = Self::parse_between_date(self.curr.as_str())?;	1✔
79	let v2 = Self::parse_between_date(self.end.as_str())?;	1✔
80	Ok(Slices::<NaiveDateTime> {	1✔
81	curr: v1,	1✔
82	end: v2,	1✔
83	increment: self.increment,	1✔
84	mode: self.mode,	1✔
85	})	1✔
86	}	1✔
87
88	fn parse_between_number(value: &str) -> BoxedResult<u64> {	×
89	let parsed = value.parse::<u64>();	×
90	match parsed {	×
91	Err(_) => throws!("Wrong value for number: {}", value),	×
92	Ok(quantity) => Ok(quantity),	×
93	}
94	}	×
95
96	fn parse_between_date(value: &str) -> BoxedResult<NaiveDateTime> {	2✔
97	if value.contains('T') {	2✔
98	let time = value.parse::<NaiveDateTime>();	1✔
99	match time {	1✔
100	Err(_) => throws!("Wrong value for datetime: {}", value),	×
101	Ok(quantity) => Ok(quantity),	1✔
102	}
103	} else {
104	let date = value.parse::<NaiveDate>();	1✔
105	match date {	1✔
106	Err(_) => throw(format!("Wrong value for date: '{}'", value)),	×
107	Ok(quantity) => quantity	1✔
108	.and_hms_opt(0, 0, 0)	1✔
109	.ok_or_else(should_fail!("Wrong value for date: '{}'", value)),	1✔
110	}
111	}
112	}	2✔
113	}
114
115	impl Slices<NaiveDateTime> {
116	fn get_interval(&self, less: i64) -> Duration {	6✔
117	let plus = self.increment.to_i64();	6✔
118	let dur = match self.mode {	6✔
119	IterateMode::Minute => Duration::minutes(plus),	×
120	IterateMode::Hour => Duration::hours(plus),	×
121	IterateMode::Day => Duration::days(plus),	6✔
122	_ => Duration::days(365),	×
123	};
124	dur - Duration::seconds(less)	6✔
125	}	6✔
126
127	fn len(&self) -> u64 {	×
128	let dur = self.end - self.curr;	×
129	let (diff, prev, div) = match self.mode {	×
130	IterateMode::Minute => (dur.num_minutes(), dur.num_seconds(), 60i64),	×
131	IterateMode::Hour => (dur.num_hours(), dur.num_minutes(), 60i64),	×
132	IterateMode::Day => (dur.num_days(), dur.num_hours(), 24i64),	×
133	_ => (1i64, 1i64, 1i64),	×
134	};
135	if diff < 0 {	×
136	0	×
137	} else {
138	let rem = prev % div;	×
139	let res = if rem == 0 { diff } else { diff + 1 };	×
140	res.to_u64()	×
141	}
142	}	×
143	}
144
145	impl Slices<u64> {
146	fn len(&self) -> u64 {	×
147	self.end - self.curr + 1	×
148	}	×
149	}
150
151	impl Iterator for Slices<u64> {
152	type Item = SliceItem;
153
154	fn next(&mut self) -> Option<Self::Item> {	13✔
155	if self.end > self.curr {	13✔
156	let next = self.curr + self.increment;	10✔
157	let last = next - 1;	10✔
158	let res = SliceItem { begin: self.curr.to_string(), end: last.to_string() };	10✔
159	self.curr = next;	10✔
160	Some(res)	10✔
161	} else {
162	None	3✔
163	}
164	}	13✔
165
166	fn size_hint(&self) -> (usize, Option<usize>) {	×
167	let num_steps = self.len();	×
168	if num_steps == 0 { (0, None) } else { (0, Some(num_steps.to_usize())) }	×
169	}	×
170	}
171
172	impl Iterator for Slices<NaiveDateTime> {
173	type Item = SliceItem;
174
175	fn next(&mut self) -> Option<Self::Item> {	4✔
176	if self.end > self.curr {	4✔
177	let last = self.curr + self.get_interval(1);	3✔
178	let part = if last < self.end { last } else { self.end };	3✔
179	let res = SliceItem { begin: format_solr_time(self.curr), end: format_solr_time(part) };	3✔
180	let next = self.get_interval(0);	3✔
181	self.curr += next;	3✔
182	Some(res)	3✔
183	} else {
184	None	1✔
185	}
186	}	4✔
187
188	fn size_hint(&self) -> (usize, Option<usize>) {	×
189	let num_steps = self.len();	×
190	if num_steps == 0 { (0, None) } else { (0, Some(num_steps.to_usize())) }	×
191	}	×
192	}
193
194	impl SliceItem {
195	pub(crate) fn filter(&self, step: Step) -> Step {	2✔
196	if self.begin.is_empty() {	2✔
197	step	×
198	} else {
199	let query =	2✔
200	replace_solr_vars(step.url.as_str(), self.begin.as_str(), self.end.as_str());	2✔
201	Step { url: query, curr: step.curr }	2✔
202	}
203	}	2✔
204	}
205
206	impl Requests {
207	pub(crate) fn len(&self) -> u64 {	×
208	let res = self.limit / self.num_docs;	×
209	if self.limit % self.num_docs == 0 { res } else { res + 1 }	×
210	}	×
211	}
212
213	impl Iterator for Requests {
214	type Item = Step;
215
216	fn next(&mut self) -> Option<Step> {	13✔
217	if self.limit > self.curr {	13✔
218	let remaining = self.limit - self.curr;	10✔
219	let rows = self.num_docs.min(remaining);	10✔
220	let query = format!("{}&start={}&rows={}", self.url, self.curr, rows);	10✔
221	let res = Step { url: query, curr: self.curr };	10✔
222	self.curr += self.num_docs;	10✔
223	Some(res)	10✔
224	} else {
225	None	3✔
226	}
227	}	13✔
228
229	fn size_hint(&self) -> (usize, Option<usize>) {	×
230	let num_steps = self.len();	×
231	if num_steps == 0 {	×
232	(0, None)	×
233	} else {
234	let max: usize = num_steps.to_usize();	×
235	(0, Some(max))	×
236	}
237	}	×
238	}
239
240	fn replace_solr_vars(query: &str, begin: &str, end: &str) -> String {	2✔
241	let query2 = replace_solr_date(query, "{begin}", begin);	2✔
242	replace_solr_date(&query2, "{end}", end)	2✔
243	}	2✔
244
245	fn format_solr_time(date_time: NaiveDateTime) -> String {	6✔
246	date_time.format("%Y-%m-%dT%H:%M:%SZ").to_string()	6✔
247	}	6✔
248
249	// endregion
250
251	// region Solr requests
252
253	impl Backup {
254	pub(crate) fn get_archive_pattern(&self, num_found: u64) -> String {	2✔
255	let prefix = match &self.archive_prefix {	2✔
256	Some(text) => text.to_string(),	×
257	None => {
258	let now: DateTime<Utc> = Utc::now();	2✔
259	let time = now.format("%Y%m%d_%H%M");	2✔
260	format!("{}_at_{}", &self.options.core, &time)	2✔
261	}
262	};
263	let ext = self.archive_compression.get_ext();	2✔
264	format!("{}_docs_{}_seq_{}.{}", prefix, num_found, BRACKETS, ext)	2✔
265	}	2✔
266
267	pub(crate) fn estimate_docs_quantity(	2✔
268	&self, schema: &SolrCore, slices: &Slices<String>,	2✔
269	) -> BoxedResult<u64> {	2✔
270	let end_limit = self.get_docs_to_retrieve(schema);	2✔
271	let num_retrieving = end_limit - self.skip;	2✔
272
273	let slice_count = slices.estimate_steps()?;	2✔
274	Ok(num_retrieving * slice_count)	2✔
275	}	2✔
276
277	pub(crate) fn get_docs_to_retrieve(&self, schema: &SolrCore) -> u64 {	7✔
278	schema.num_found.min(self.limit.unwrap_or(u64::MAX))	7✔
279	}	7✔
280
281	pub(crate) fn get_steps(&self, schema: &SolrCore) -> Requests {	3✔
282	let include_hash: HashSet<String> = HashSet::from_iter(schema.fields.clone());	3✔
283	let exclude_hash: HashSet<String> = HashSet::from_iter(self.exclude.clone());	3✔
284	let diff: Vec<String> = include_hash.difference(&exclude_hash).map(String::from).collect();	3✔
285
286	debug!("Include fields {:?}", include_hash);	3✔
287	debug!("Exclude fields {:?}", exclude_hash);	3✔
288	debug!("Actual fields {:?}", diff);	3✔
289
290	let fl = self.get_query_fields(diff);	3✔
291	let query = self.get_query_url(&fl, true);	3✔
292	let end_limit = self.get_docs_to_retrieve(schema);	3✔
293	Requests { curr: self.skip, limit: end_limit, num_docs: self.num_docs, url: query }	3✔
294	}	3✔
295
296	pub(crate) fn get_query_fields(&self, core_fields: Vec<String>) -> String {	3✔
297	if core_fields.is_empty() {	3✔
UNCOV 298	EMPTY_STRING	×
299	} else {
300	let all = core_fields.join(COMMA);	3✔
301	"&fl=".append(&all)	3✔
302	}
303	}	3✔
304
305	pub(crate) fn get_query_for_diagnostics(&self) -> String {	2✔
306	let url = self.get_query_url(EMPTY_STR, false);	2✔
307	format!("{}&start=0&rows=1", url)	2✔
308	}	2✔
309
310	pub(crate) fn replace_vars(&self, query: &str, raw: bool) -> String {	6✔
311	if raw \|\| self.iterate_between.is_empty() {	6✔
312	query.to_string()	6✔
313	} else {
314	let (begin, end) = self.get_between();	×
315	replace_solr_vars(query, begin, end)	×
316	}
317	}	6✔
318
319	pub(crate) fn get_query_url(&self, selected: &str, raw: bool) -> String {	6✔
320	let qparam = self.query.as_deref().unwrap_or(":");	6✔
321	let qfixed = self.replace_vars(qparam, raw);	6✔
322	let filterq = solr_query(&qfixed);	6✔
323	let fqparam = self.fq.as_deref().unwrap_or(":");	6✔
324	let filterfq = solr_query(fqparam);	6✔
325
326	let sort: String = if self.order.is_empty() {	6✔
327	EMPTY_STRING	4✔
328	} else {
329	let all: Vec<String> = self.order.iter().map(\|field\| field.to_string()).collect();	6✔
330	let joined = all.join(COMMA);	2✔
331	"&sort=".append(&joined)	2✔
332	};
333	let parts = [	6✔
334	self.options.url.with_suffix("/"),	6✔
335	self.options.core.clone(),	6✔
336	"/select?wt=json&indent=off&omitHeader=true".to_string(),	6✔
337	format!("&q={}", filterq),	6✔
338	format!("&fq={}", filterfq),	6✔
339	sort,	6✔
340	self.transfer.get_param("&"),	6✔
341	selected.to_string(),	6✔
342	];	6✔
343	parts.concat()	6✔
344	}	6✔
345
346	pub(crate) fn get_slices(&self) -> Slices<String> {	2✔
347	let (begin, end) = self.get_between();	2✔
348	Slices::<String> {	2✔
349	curr: begin.to_string(),	2✔
350	end: end.to_string(),	2✔
351	increment: self.iterate_step,	2✔
352	mode: self.iterate_by,	2✔
353	}	2✔
354	}	2✔
355
356	fn get_between(&self) -> (&str, &str) {	2✔
357	if self.iterate_between.is_empty() {	2✔
358	(EMPTY_STR, EMPTY_STR)	2✔
359	} else {
360	(self.iterate_between[0].as_str(), self.iterate_between[1].as_str())	×
361	}
362	}	2✔
363	}
364
365	// endregion
366
367	#[cfg(test)]
368	mod tests {
369	// region mockup
370
371	use crate::{
372	args::{Backup, Cli, Commands, IterateMode, shared::TEST_SELECT_FIELDS},
373	fails::{BoxedResult, raise},
374	helpers::{COMMA, EMPTY_STR},
375	steps::{Slices, SolrCore},
376	};
377	use pretty_assertions::assert_eq;
378
379	impl Commands {
380	pub(crate) fn get(&self) -> BoxedResult<&Backup> {	1✔
381	match &self {	1✔
382	Self::Backup(gets) => Ok(&gets),	1✔
383	_ => raise("command must be 'backup' !"),	×
384	}
385	}	1✔
386	}
387
388	impl SolrCore {
389	pub(crate) fn mockup() -> Self {	1✔
390	SolrCore { num_found: 100, fields: vec![TEST_SELECT_FIELDS.split(COMMA).collect()] }	1✔
391	}	1✔
392	}
393
394	// endregion
395
396	// region iterators
397
398	#[test]
399	fn check_iterator_for_params_get() {	1✔
400	let parsed = Cli::mockup_args_backup();	1✔
401	let gets = parsed.get().unwrap();	1✔
402	let core_info = SolrCore::mockup();	1✔
403	let query = gets.get_query_url(EMPTY_STR, true);	1✔
404
405	let mut i = 0;	1✔
406	for step in gets.get_steps(&core_info) {	8✔
407	let url = step.url;	8✔
408	assert_eq!(url.is_empty(), false);	8✔
409	assert_eq!(url.starts_with(&query), true);	8✔
410	i += 1;	8✔
411	}
412	assert_eq!(i, 8);	1✔
413	}	1✔
414
415	#[test]
416	fn check_iterator_for_slices_u64() {	1✔
417	let slices = Slices::<String>::get_slice_of(16, 2);	1✔
418	for step in slices {	9✔
419	assert!(step.begin < step.end, "# {} -> {}", step.begin, step.end)	8✔
420	}
421	}	1✔
422
423	#[test]
424	fn check_iterator_for_slices_datetime() {	1✔
425	let src = Slices::<String> {	1✔
426	curr: "2020-04-01".to_string(),	1✔
427	end: "2020-04-03T11:12:13".to_string(),	1✔
428	increment: 1,	1✔
429	mode: IterateMode::Day,	1✔
430	};	1✔
431
432	let slices = src.get_period_slices();	1✔
433	assert!(slices.is_ok());	1✔
434
435	if let Ok(seq) = slices {	1✔
436	for step in seq {	4✔
437	assert!(step.begin < step.end, "# {} -> {}", step.begin, step.end)	3✔
438	}
439	}	×
440	}	1✔
441
442	// endregion
443	}

juarezr / solrcopy / 18363067970

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous