17814558627

Committed 18 Sep 2025 12:47AM UTC coverage: 71.375% (-0.3%) from 71.626%

Build # 17814558627

Build Type

push

github

Committed by

juarezr

Commit Message

Update README.md

Run Details

1339 of 1876 relevant lines covered (71.38%)

2.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.06

/src/fetch.rs

use super::{args::Backup, connection::SolrClient, fails::*, helpers::*, steps::SolrCore};
use log::debug;
use regex::Regex;

// region Solr Core

impl Backup {
    pub(crate) fn inspect_core(&self) -> BoxedResult<SolrCore> {
        let diagnostics_query_url = self.get_query_for_diagnostics();
        debug!("Inspecting schema of core {} at: {}", self.options.core, diagnostics_query_url);

        // try sometimes for finding the greatest num_found of docs answered by the core
        // Used for fixing problems with corrupted replicas of cores with more than 1 shard
        let times = (self.workaround_shards * 5) + 1;

        let mut res = SolrCore { num_found: 0, fields: vec![] };
        for it in 0..times {
            let json = SolrClient::query_get_as_text(&diagnostics_query_url)?;
            if let Ok(next) = SolrCore::parse_core_schema(self, &json) {
                debug!("#{} Solr query returned num_found: {}", it, next.num_found);
                if next.num_found > res.num_found {
                    res = next;
                }
            }
        }
        if res.num_found <= self.skip && self.skip > 0 {
            throw(format!(
                "Requested {} in --skip but found {} docs with the query.",
                self.skip, res.num_found
            ))?;
        }
        debug!("Core schema: {:?}", res);
        Ok(res)
    }
}

impl SolrCore {
    fn parse_core_schema(gets: &Backup, json: &str) -> BoxedResult<Self> {
        let core_name = &gets.options.core;

        let total_docs = Self::parse_num_found(json)?;
        if total_docs < 1 {
            throw(format!("Solr Core '{}'is empty!", core_name))?
        };
        let parsed_fields = Self::parse_field_names(json);

        let core_fields = if gets.select.is_empty() {
            match parsed_fields {
                None => throw(format!("Missing fields to parse in Solr Core '{}'!", core_name))?,
                Some(fields) => fields,
            }
        } else {
            // TODO: check if args.select fields matches parsed_fields when --validate
            gets.select.clone()
        };
        let res = SolrCore { num_found: total_docs, fields: core_fields };
        Ok(res)
    }

    pub(crate) fn parse_num_found(json: &str) -> BoxedResult<u64> {
        lazy_static! {
            static ref REGNF: Regex = Regex::new("\"numFound\":(\\d+),").unwrap();
        }
        match REGNF.get_group(json, 1) {
            None => throw(format!("Error parsing numFound from solr query: {}", json))?,
            Some(group1) => {
                let res = group1.parse::<u64>();
                res.or_else(|_| {
                    throw::<u64>(format!("Error converting numFound from solr query: {}", json))
                })
            }
        }
    }

    fn parse_field_names(json: &str) -> Option<Vec<String>> {
        lazy_static! {
            static ref REGFN: Regex = Regex::new("\"(\\w+)\":").unwrap();
        }
        let row1 = Self::parse_docs_from_query(json)?;

        let matches = REGFN.get_group_values(row1, 1);
        let filtered = matches
            .iter()
            .filter(|s| !s.starts_with('_'))
            .map(|&s| s.to_string())
            .collect::<Vec<String>>();
        Some(filtered)
    }

    /// Strips out: `[{  "a": "b", "c": "d" }]` from Solr json response
    /// ``` json
    /// {"response":{"numFound":46,"start":0,"docs":_____}}
    /// ```
    pub(crate) fn parse_docs_from_query(json: &str) -> Option<&str> {
        json.find_text_between("docs\":", "}}") // -> [{  ... }]
    }
}

// endregion

#[cfg(test)]
mod tests {
    use super::{SolrCore, StringHelpers};
    use pretty_assertions::assert_eq;

    const CORE_1ROW: &str = r#"{
        "response":{"numFound":46,"start":0,
            "docs":[
                {
                "id":"3007WFP",
                "name":["Dell Widescreen UltraSharp 3007WFP"],
                "cat":["electronics and computer1"],
                "price":[2199.0]}
            ]}}"#;
    const CORE_3ROW: &str = r#"{"response":{"numFound":46,"start":0,
            "docs":[
                {"id":"3007WFP","name":["Dell Widescreen UltraSharp 3007WFP"],"cat":["electronics and computer1"],"price":[2199.0]},
                {"id":"100-435805","name":["ATI Radeon X1900 XTX 512 MB PCIE Video Card"],"cat":["electronics","graphics card"],"price":[649.99]},
                {"id":"EN7800GTX/2DHTV/256M","name":["ASUS Extreme N7800GTX/2DHTV (256 MB)"],"cat":["electronics","graphics card"],"price":[479.95]}
            ]}}"#;

    #[test]
    fn check_schema_num_found() {
        let num_found = SolrCore::parse_num_found(CORE_1ROW);
        assert_eq!(num_found.ok(), Some(46));
    }

    #[test]
    fn check_schema_fields() {
        let fields = SolrCore::parse_field_names(CORE_1ROW);
        assert_eq!(fields.is_some(), true);

        let fields2 = fields.unwrap();

        assert_eq!(fields2.len(), 4);
        assert_eq!(fields2.get(0).unwrap(), "id");
        assert_eq!(fields2.get(1).unwrap(), "name");
        assert_eq!(fields2.get(2).unwrap(), "cat");
        assert_eq!(fields2.get(3).unwrap(), "price");
    }

    #[test]
    fn check_query_docs() {
        let docs = SolrCore::parse_docs_from_query(CORE_3ROW);
        assert_eq!(docs.is_some(), true);

        let json = docs.unwrap().remove_whitespace();

        let starting = &json[..2];
        assert_eq!(starting, "[{");

        let two = json.len() - 2;
        let ending = &json[two..];
        assert_eq!(ending, "}]");

        let rows = json.split("},{").collect::<Vec<&str>>();
        assert_eq!(rows.len(), 3);
    }
}

1	use super::{args::Backup, connection::SolrClient, fails::, helpers::, steps::SolrCore};
2	use log::debug;
3	use regex::Regex;
4
5	// region Solr Core
6
7	impl Backup {
8	pub(crate) fn inspect_core(&self) -> BoxedResult<SolrCore> {	1✔
9	let diagnostics_query_url = self.get_query_for_diagnostics();	1✔
10	debug!("Inspecting schema of core {} at: {}", self.options.core, diagnostics_query_url);	1✔
11
12	// try sometimes for finding the greatest num_found of docs answered by the core
13	// Used for fixing problems with corrupted replicas of cores with more than 1 shard
14	let times = (self.workaround_shards * 5) + 1;	1✔
15
16	let mut res = SolrCore { num_found: 0, fields: vec![] };	1✔
17	for it in 0..times {	1✔
18	let json = SolrClient::query_get_as_text(&diagnostics_query_url)?;	1✔
19	if let Ok(next) = SolrCore::parse_core_schema(self, &json) {	1✔
20	debug!("#{} Solr query returned num_found: {}", it, next.num_found);	1✔
21	if next.num_found > res.num_found {	1✔
22	res = next;	1✔
23	}	1✔
24	}	×
25	}
26	if res.num_found <= self.skip && self.skip > 0 {	1✔
27	throw(format!(	×
28	"Requested {} in --skip but found {} docs with the query.",	×
29	self.skip, res.num_found	×
30	))?;	×
31	}	1✔
32	debug!("Core schema: {:?}", res);	1✔
33	Ok(res)	1✔
34	}	1✔
35	}
36
37	impl SolrCore {
38	fn parse_core_schema(gets: &Backup, json: &str) -> BoxedResult<Self> {	1✔
39	let core_name = &gets.options.core;	1✔
40
41	let total_docs = Self::parse_num_found(json)?;	1✔
42	if total_docs < 1 {	1✔
43	throw(format!("Solr Core '{}'is empty!", core_name))?	×
44	};	1✔
45	let parsed_fields = Self::parse_field_names(json);	1✔
46
47	let core_fields = if gets.select.is_empty() {	1✔
48	match parsed_fields {	1✔
49	None => throw(format!("Missing fields to parse in Solr Core '{}'!", core_name))?,	×
50	Some(fields) => fields,	1✔
51	}
52	} else {
53	// TODO: check if args.select fields matches parsed_fields when --validate
54	gets.select.clone()	×
55	};
56	let res = SolrCore { num_found: total_docs, fields: core_fields };	1✔
57	Ok(res)	1✔
58	}	1✔
59
60	pub(crate) fn parse_num_found(json: &str) -> BoxedResult<u64> {	2✔
61	lazy_static! {
62	static ref REGNF: Regex = Regex::new("\"numFound\":(\\d+),").unwrap();
63	}
64	match REGNF.get_group(json, 1) {	2✔
65	None => throw(format!("Error parsing numFound from solr query: {}", json))?,	×
66	Some(group1) => {	2✔
67	let res = group1.parse::<u64>();	2✔
68	res.or_else(\|_\| {	2✔
69	throw::<u64>(format!("Error converting numFound from solr query: {}", json))	×
70	})	×
71	}
72	}
73	}	2✔
74
75	fn parse_field_names(json: &str) -> Option<Vec<String>> {	2✔
76	lazy_static! {
77	static ref REGFN: Regex = Regex::new("\"(\\w+)\":").unwrap();
78	}
79	let row1 = Self::parse_docs_from_query(json)?;	2✔
80
81	let matches = REGFN.get_group_values(row1, 1);	2✔
82	let filtered = matches	2✔
83	.iter()	2✔
84	.filter(\|s\| !s.starts_with('_'))	16✔
85	.map(\|&s\| s.to_string())	14✔
86	.collect::<Vec<String>>();	2✔
87	Some(filtered)	2✔
88	}	2✔
89
90	/// Strips out: `[{ "a": "b", "c": "d" }]` from Solr json response
91	/// ``` json
92	/// {"response":{"numFound":46,"start":0,"docs":_____}}
93	/// ```
94	pub(crate) fn parse_docs_from_query(json: &str) -> Option<&str> {	4✔
95	json.find_text_between("docs\":", "}}") // -> [{ ... }]	4✔
96	}	4✔
97	}
98
99	// endregion
100
101	#[cfg(test)]
102	mod tests {
103	use super::{SolrCore, StringHelpers};
104	use pretty_assertions::assert_eq;
105
106	const CORE_1ROW: &str = r#"{
107	"response":{"numFound":46,"start":0,
108	"docs":[
109	{
110	"id":"3007WFP",
111	"name":["Dell Widescreen UltraSharp 3007WFP"],
112	"cat":["electronics and computer1"],
113	"price":[2199.0]}
114	]}}"#;
115	const CORE_3ROW: &str = r#"{"response":{"numFound":46,"start":0,
116	"docs":[
117	{"id":"3007WFP","name":["Dell Widescreen UltraSharp 3007WFP"],"cat":["electronics and computer1"],"price":[2199.0]},
118	{"id":"100-435805","name":["ATI Radeon X1900 XTX 512 MB PCIE Video Card"],"cat":["electronics","graphics card"],"price":[649.99]},
119	{"id":"EN7800GTX/2DHTV/256M","name":["ASUS Extreme N7800GTX/2DHTV (256 MB)"],"cat":["electronics","graphics card"],"price":[479.95]}
120	]}}"#;
121
122	#[test]
123	fn check_schema_num_found() {	1✔
124	let num_found = SolrCore::parse_num_found(CORE_1ROW);	1✔
125	assert_eq!(num_found.ok(), Some(46));	1✔
126	}	1✔
127
128	#[test]
129	fn check_schema_fields() {	1✔
130	let fields = SolrCore::parse_field_names(CORE_1ROW);	1✔
131	assert_eq!(fields.is_some(), true);	1✔
132
133	let fields2 = fields.unwrap();	1✔
134
135	assert_eq!(fields2.len(), 4);	1✔
136	assert_eq!(fields2.get(0).unwrap(), "id");	1✔
137	assert_eq!(fields2.get(1).unwrap(), "name");	1✔
138	assert_eq!(fields2.get(2).unwrap(), "cat");	1✔
139	assert_eq!(fields2.get(3).unwrap(), "price");	1✔
140	}	1✔
141
142	#[test]
143	fn check_query_docs() {	1✔
144	let docs = SolrCore::parse_docs_from_query(CORE_3ROW);	1✔
145	assert_eq!(docs.is_some(), true);	1✔
146
147	let json = docs.unwrap().remove_whitespace();	1✔
148
149	let starting = &json[..2];	1✔
150	assert_eq!(starting, "[{");	1✔
151
152	let two = json.len() - 2;	1✔
153	let ending = &json[two..];	1✔
154	assert_eq!(ending, "}]");	1✔
155
156	let rows = json.split("},{").collect::<Vec<&str>>();	1✔
157	assert_eq!(rows.len(), 3);	1✔
158	}	1✔
159	}

juarezr / solrcopy / 17814558627

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous