22975356339

Committed 11 Mar 2026 09:28PM UTC coverage: 81.975% (+0.2%) from 81.743%

Build # 22975356339

Build Type

Pull #45

github

Committed by

web-flow

Commit Message

Merge 32c30b775 into 4de9ef34c

Pull Request Pull Request #45: Bug fixes and improved testing

Coverage Stats

276 of 310 new or added lines in 13 files covered. (89.03%)

8 existing lines in 5 files now uncovered.

1901 of 2319 relevant lines covered (81.97%)

3.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.88

/src/fetch.rs

use super::{args::Backup, connection::SolrClient, fails::*, helpers::*, models::SolrCore};
use log::{debug, trace};
use regex::Regex;

// region Solr Core

impl Backup {
    pub(crate) fn inspect_core(&self) -> BoxedResult<SolrCore> {
        let diagnostics_query_url = self.get_query_for_diagnostics();
        debug!("Inspecting schema of core {} at: {}", self.options.core, diagnostics_query_url);

        // try sometimes for finding the greatest num_found of docs answered by the core
        // Used for fixing problems with corrupted replicas of cores with more than 1 shard
        let times = (self.workaround_shards * 5) + 1;

        let mut res = SolrCore { num_found: 0, fields: vec![] };
        for it in 0..times {
            let json = SolrClient::send_get_as_json(&diagnostics_query_url)?;
            if let Ok(next) = SolrCore::parse_core_schema(self, &json) {
                trace!("#{} Solr query returned num_found: {}", it, next.num_found);
                if next.num_found > res.num_found {
                    res = next;
                }
            }
        }
        if res.num_found <= self.skip && self.skip > 0 {
            throw(format!(
                "Requested {} in --skip but found {} docs with the query.",
                self.skip, res.num_found
            ))?;
        }
        debug!("Core schema: {:?}", res);
        Ok(res)
    }

    pub(crate) fn query_num_found(&self, begin: &str, end: &str) -> BoxedResult<u64> {
        // try sometimes for finding the greatest num_found of docs answered by the core
        // Used for fixing problems with corrupted replicas of cores with more than 1 shard
        let times = (self.workaround_shards * 5) + 1;
        let mut prev_num_found = 0;
        let query_url = self.get_query_num_found(begin, end);
        for it in 0..times {
            let json = SolrClient::send_get_as_json(&query_url)?;
            if let Ok(num_found) = SolrCore::parse_num_found(&json) {
                debug!(
                    "#{} Solr query returned num_found={} for range {} to {}",
                    it, num_found, begin, end
                );
                if num_found > prev_num_found {
                    prev_num_found = num_found;
                }
            }
        }
        Ok(prev_num_found)
    }
}

impl SolrCore {
    fn parse_core_schema(gets: &Backup, json: &str) -> BoxedResult<Self> {
        let core_name = &gets.options.core;

        let total_docs = Self::parse_num_found(json)?;
        if total_docs < 1 {
            throw(format!("Solr Core '{}'is empty!", core_name))?
        };
        let parsed_fields = Self::parse_field_names(json);

        let core_fields = if gets.select.is_empty() {
            match parsed_fields {
                None => throw(format!("Missing fields to parse in Solr Core '{}'!", core_name))?,
                Some(fields) => fields,
            }
        } else {
            // TODO: check if args.select fields matches parsed_fields when --validate
            gets.select.clone()
        };
        let res = SolrCore { num_found: total_docs, fields: core_fields };
        Ok(res)
    }

    pub(crate) fn parse_num_found(json: &str) -> BoxedResult<u64> {
        lazy_static! {
            static ref REGNF: Regex = Regex::new("\"numFound\":(\\d+),").unwrap();
        }
        match REGNF.get_group(json, 1) {
            None => throw(format!("Error parsing numFound from solr query: {}", json))?,
            Some(group1) => {
                let res = group1.parse::<u64>();
                res.or_else(|_| {
                    throw::<u64>(format!("Error converting numFound from solr query: {}", json))
                })
            }
        }
    }

    fn parse_field_names(json: &str) -> Option<Vec<String>> {
        lazy_static! {
            static ref REGFN: Regex = Regex::new("\"(\\w+)\":").unwrap();
        }
        let row1 = Self::parse_docs_from_query(json)?;

        let matches = REGFN.get_group_values(row1, 1);
        let filtered = matches
            .iter()
            .filter(|s| !s.starts_with('_'))
            .map(|&s| s.to_string())
            .collect::<Vec<String>>();
        Some(filtered)
    }

    /// Strips out: `[{  "a": "b", "c": "d" }]` from Solr json response
    /// ``` json
    /// {"response":{"numFound":46,"start":0,"docs":_____}}
    /// ```
    pub(crate) fn parse_docs_from_query(json: &str) -> Option<&str> {
        json.find_text_between("docs\":", "}}") // -> [{  ... }]
    }
}

// endregion

#[cfg(test)]
mod tests {
    use super::{SolrCore, StringHelpers};
    use pretty_assertions::assert_eq;

    const CORE_1ROW: &str = r#"{
        "response":{"numFound":46,"start":0,
            "docs":[
                {
                "id":"3007WFP",
                "name":["Dell Widescreen UltraSharp 3007WFP"],
                "cat":["electronics and computer1"],
                "price":[2199.0]}
            ]}}"#;
    const CORE_3ROW: &str = r#"{"response":{"numFound":46,"start":0,
            "docs":[
                {"id":"3007WFP","name":["Dell Widescreen UltraSharp 3007WFP"],"cat":["electronics and computer1"],"price":[2199.0]},
                {"id":"100-435805","name":["ATI Radeon X1900 XTX 512 MB PCIE Video Card"],"cat":["electronics","graphics card"],"price":[649.99]},
                {"id":"EN7800GTX/2DHTV/256M","name":["ASUS Extreme N7800GTX/2DHTV (256 MB)"],"cat":["electronics","graphics card"],"price":[479.95]}
            ]}}"#;

    #[test]
    fn check_schema_num_found() {
        let num_found = SolrCore::parse_num_found(CORE_1ROW);
        assert_eq!(num_found.ok(), Some(46));
    }

    #[test]
    fn check_schema_fields() {
        let fields = SolrCore::parse_field_names(CORE_1ROW);
        assert_eq!(fields.is_some(), true);

        let fields2 = fields.unwrap();

        assert_eq!(fields2.len(), 4);
        assert_eq!(fields2.get(0).unwrap(), "id");
        assert_eq!(fields2.get(1).unwrap(), "name");
        assert_eq!(fields2.get(2).unwrap(), "cat");
        assert_eq!(fields2.get(3).unwrap(), "price");
    }

    #[test]
    fn check_query_docs() {
        let docs = SolrCore::parse_docs_from_query(CORE_3ROW);
        assert_eq!(docs.is_some(), true);

        let json = docs.unwrap().remove_whitespace();

        let starting = &json[..2];
        assert_eq!(starting, "[{");

        let two = json.len() - 2;
        let ending = &json[two..];
        assert_eq!(ending, "}]");

        let rows = json.split("},{").collect::<Vec<&str>>();
        assert_eq!(rows.len(), 3);
    }
}

1	use super::{args::Backup, connection::SolrClient, fails::, helpers::, models::SolrCore};
2	use log::{debug, trace};
3	use regex::Regex;
4
5	// region Solr Core
6
7	impl Backup {
8	pub(crate) fn inspect_core(&self) -> BoxedResult<SolrCore> {	2✔
9	let diagnostics_query_url = self.get_query_for_diagnostics();	2✔
10	debug!("Inspecting schema of core {} at: {}", self.options.core, diagnostics_query_url);	2✔
11
12	// try sometimes for finding the greatest num_found of docs answered by the core
13	// Used for fixing problems with corrupted replicas of cores with more than 1 shard
14	let times = (self.workaround_shards * 5) + 1;	2✔
15
16	let mut res = SolrCore { num_found: 0, fields: vec![] };	2✔
17	for it in 0..times {	2✔
18	let json = SolrClient::send_get_as_json(&diagnostics_query_url)?;	2✔
19	if let Ok(next) = SolrCore::parse_core_schema(self, &json) {	2✔
20	trace!("#{} Solr query returned num_found: {}", it, next.num_found);	2✔
21	if next.num_found > res.num_found {	2✔
22	res = next;	2✔
23	}	2✔
24	}	×
25	}
26	if res.num_found <= self.skip && self.skip > 0 {	2✔
27	throw(format!(	×
28	"Requested {} in --skip but found {} docs with the query.",	×
29	self.skip, res.num_found	×
30	))?;	×
31	}	2✔
32	debug!("Core schema: {:?}", res);	2✔
33	Ok(res)	2✔
34	}	2✔
35
36	pub(crate) fn query_num_found(&self, begin: &str, end: &str) -> BoxedResult<u64> {	2✔
37	// try sometimes for finding the greatest num_found of docs answered by the core
38	// Used for fixing problems with corrupted replicas of cores with more than 1 shard
39	let times = (self.workaround_shards * 5) + 1;	2✔
40	let mut prev_num_found = 0;	2✔
41	let query_url = self.get_query_num_found(begin, end);	2✔
42	for it in 0..times {	2✔
43	let json = SolrClient::send_get_as_json(&query_url)?;	2✔
44	if let Ok(num_found) = SolrCore::parse_num_found(&json) {	2✔
45	debug!(	2✔
46	"#{} Solr query returned num_found={} for range {} to {}",
47	it, num_found, begin, end
48	);
49	if num_found > prev_num_found {	2✔
50	prev_num_found = num_found;	2✔
51	}	2✔
NEW 52	}	×
53	}
54	Ok(prev_num_found)	2✔
55	}	2✔
56	}
57
58	impl SolrCore {
59	fn parse_core_schema(gets: &Backup, json: &str) -> BoxedResult<Self> {	2✔
60	let core_name = &gets.options.core;	2✔
61
62	let total_docs = Self::parse_num_found(json)?;	2✔
63	if total_docs < 1 {	2✔
64	throw(format!("Solr Core '{}'is empty!", core_name))?	×
65	};	2✔
66	let parsed_fields = Self::parse_field_names(json);	2✔
67
68	let core_fields = if gets.select.is_empty() {	2✔
69	match parsed_fields {	2✔
70	None => throw(format!("Missing fields to parse in Solr Core '{}'!", core_name))?,	×
71	Some(fields) => fields,	2✔
72	}
73	} else {
74	// TODO: check if args.select fields matches parsed_fields when --validate
75	gets.select.clone()	×
76	};
77	let res = SolrCore { num_found: total_docs, fields: core_fields };	2✔
78	Ok(res)	2✔
79	}	2✔
80
81	pub(crate) fn parse_num_found(json: &str) -> BoxedResult<u64> {	5✔
82	lazy_static! {
83	static ref REGNF: Regex = Regex::new("\"numFound\":(\\d+),").unwrap();
84	}
85	match REGNF.get_group(json, 1) {	5✔
86	None => throw(format!("Error parsing numFound from solr query: {}", json))?,	×
87	Some(group1) => {	5✔
88	let res = group1.parse::<u64>();	5✔
89	res.or_else(\|_\| {	5✔
90	throw::<u64>(format!("Error converting numFound from solr query: {}", json))	×
91	})	×
92	}
93	}
94	}	5✔
95
96	fn parse_field_names(json: &str) -> Option<Vec<String>> {	3✔
97	lazy_static! {
98	static ref REGFN: Regex = Regex::new("\"(\\w+)\":").unwrap();
99	}
100	let row1 = Self::parse_docs_from_query(json)?;	3✔
101
102	let matches = REGFN.get_group_values(row1, 1);	3✔
103	let filtered = matches	3✔
104	.iter()	3✔
105	.filter(\|s\| !s.starts_with('_'))	34✔
106	.map(\|&s\| s.to_string())	30✔
107	.collect::<Vec<String>>();	3✔
108	Some(filtered)	3✔
109	}	3✔
110
111	/// Strips out: `[{ "a": "b", "c": "d" }]` from Solr json response
112	/// ``` json
113	/// {"response":{"numFound":46,"start":0,"docs":_____}}
114	/// ```
115	pub(crate) fn parse_docs_from_query(json: &str) -> Option<&str> {	6✔
116	json.find_text_between("docs\":", "}}") // -> [{ ... }]	6✔
117	}	6✔
118	}
119
120	// endregion
121
122	#[cfg(test)]
123	mod tests {
124	use super::{SolrCore, StringHelpers};
125	use pretty_assertions::assert_eq;
126
127	const CORE_1ROW: &str = r#"{
128	"response":{"numFound":46,"start":0,
129	"docs":[
130	{
131	"id":"3007WFP",
132	"name":["Dell Widescreen UltraSharp 3007WFP"],
133	"cat":["electronics and computer1"],
134	"price":[2199.0]}
135	]}}"#;
136	const CORE_3ROW: &str = r#"{"response":{"numFound":46,"start":0,
137	"docs":[
138	{"id":"3007WFP","name":["Dell Widescreen UltraSharp 3007WFP"],"cat":["electronics and computer1"],"price":[2199.0]},
139	{"id":"100-435805","name":["ATI Radeon X1900 XTX 512 MB PCIE Video Card"],"cat":["electronics","graphics card"],"price":[649.99]},
140	{"id":"EN7800GTX/2DHTV/256M","name":["ASUS Extreme N7800GTX/2DHTV (256 MB)"],"cat":["electronics","graphics card"],"price":[479.95]}
141	]}}"#;
142
143	#[test]
144	fn check_schema_num_found() {	1✔
145	let num_found = SolrCore::parse_num_found(CORE_1ROW);	1✔
146	assert_eq!(num_found.ok(), Some(46));	1✔
147	}	1✔
148
149	#[test]
150	fn check_schema_fields() {	1✔
151	let fields = SolrCore::parse_field_names(CORE_1ROW);	1✔
152	assert_eq!(fields.is_some(), true);	1✔
153
154	let fields2 = fields.unwrap();	1✔
155
156	assert_eq!(fields2.len(), 4);	1✔
157	assert_eq!(fields2.get(0).unwrap(), "id");	1✔
158	assert_eq!(fields2.get(1).unwrap(), "name");	1✔
159	assert_eq!(fields2.get(2).unwrap(), "cat");	1✔
160	assert_eq!(fields2.get(3).unwrap(), "price");	1✔
161	}	1✔
162
163	#[test]
164	fn check_query_docs() {	1✔
165	let docs = SolrCore::parse_docs_from_query(CORE_3ROW);	1✔
166	assert_eq!(docs.is_some(), true);	1✔
167
168	let json = docs.unwrap().remove_whitespace();	1✔
169
170	let starting = &json[..2];	1✔
171	assert_eq!(starting, "[{");	1✔
172
173	let two = json.len() - 2;	1✔
174	let ending = &json[two..];	1✔
175	assert_eq!(ending, "}]");	1✔
176
177	let rows = json.split("},{").collect::<Vec<&str>>();	1✔
178	assert_eq!(rows.len(), 3);	1✔
179	}	1✔
180	}

juarezr / solrcopy / 22975356339

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous