13958601093

Committed 19 Mar 2025 04:17PM UTC coverage: 74.164% (-1.5%) from 75.71%

Build # 13958601093

Build Type

push

github

Committed by

web-flow

Commit Message

Clean up properties docs (#6315)

Run Details

58056 of 78281 relevant lines covered (74.16%)

819371.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.93

/provider/source/src/cldr_cache.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

#![allow(dead_code)] // features

use crate::cldr_serde::eras::EraData;
use crate::datetime::DatagenCalendar;
use crate::source::SerdeCache;
use crate::CoverageLevel;
use icu::locale::provider::{
    LocaleLikelySubtagsExtendedV1, LocaleLikelySubtagsLanguageV1, LocaleLikelySubtagsScriptRegionV1,
};
use icu::locale::LocaleExpander;
use icu_provider::prelude::*;
use icu_provider::DataError;
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::fmt::Debug;
use std::str::FromStr;
use std::sync::OnceLock;
use writeable::Writeable;

#[derive(Debug)]
pub(crate) struct CldrCache {
    pub(crate) serde_cache: SerdeCache,
    dir_suffix: OnceLock<Result<&'static str, DataError>>,
    extended_locale_expander: OnceLock<Result<LocaleExpander, DataError>>,
    #[allow(clippy::type_complexity)]
    pub(crate) calendar_eras:
        OnceLock<Result<BTreeMap<DatagenCalendar, Vec<(usize, EraData)>>, DataError>>,
    #[cfg(feature = "experimental")]
    // used by transforms/mod.rs
    pub(crate) transforms: OnceLock<
        Result<std::sync::Mutex<icu::experimental::transliterate::RuleCollection>, DataError>,
    >,
    pub(crate) tz_caches: crate::time_zones::Caches,
}

impl CldrCache {
    pub(crate) fn from_serde_cache(serde_cache: SerdeCache) -> Self {
        CldrCache {
            serde_cache,
            dir_suffix: Default::default(),
            extended_locale_expander: Default::default(),
            calendar_eras: Default::default(),
            #[cfg(feature = "experimental")]
            transforms: Default::default(),
            tz_caches: Default::default(),
        }
    }

    pub(crate) fn core(&self) -> CldrDirNoLang<'_> {
        CldrDirNoLang(self, "cldr-core".to_owned())
    }

    pub(crate) fn numbers(&self) -> CldrDirLang<'_> {
        CldrDirLang(self, "cldr-numbers".to_owned())
    }

    pub(crate) fn misc(&self) -> CldrDirLang<'_> {
        CldrDirLang(self, "cldr-misc".to_owned())
    }

    pub(crate) fn bcp47(&self) -> CldrDirNoLang<'_> {
        CldrDirNoLang(self, "cldr-bcp47/bcp47".to_string())
    }

    pub(crate) fn personnames(&self) -> CldrDirLang<'_> {
        CldrDirLang(self, "cldr-person-names".to_owned())
    }

    pub(crate) fn displaynames(&self) -> CldrDirLang<'_> {
        CldrDirLang(self, "cldr-localenames".to_owned())
    }

    pub(crate) fn units(&self) -> CldrDirLang<'_> {
        CldrDirLang(self, "cldr-units".to_owned())
    }

    pub(crate) fn dates(&self, cal: &str) -> CldrDirLang<'_> {
        CldrDirLang(
            self,
            if cal == "gregorian" || cal == "generic" {
                "cldr-dates".to_owned()
            } else {
                format!("cldr-cal-{cal}")
            },
        )
    }

    pub(crate) fn locales(
        &self,
        levels: impl IntoIterator<Item = CoverageLevel>,
    ) -> Result<Vec<DataLocale>, DataError> {
        let levels = levels.into_iter().collect::<HashSet<_>>();
        let mut locales: Vec<DataLocale> = self
            .serde_cache
            .read_and_parse_json::<crate::cldr_serde::coverage_levels::Resource>(
                "cldr-core/coverageLevels.json",
            )?
            .coverage_levels
            .iter()
            .filter_map(|(locale, c)| levels.contains(c).then_some(locale))
            .cloned()
            .map(Into::into)
            // `und` needs to be part of every set
            .chain([Default::default()])
            .collect();
        locales.sort_by(|a, b| {
            let b = b.write_to_string();
            a.strict_cmp(b.as_bytes())
        });
        Ok(locales)
    }

    pub(crate) fn dir_suffix(&self) -> Result<&'static str, DataError> {
        *self.dir_suffix.get_or_init(|| {
            if self.serde_cache.list("cldr-misc-full")?.next().is_some() {
                Ok("full")
            } else {
                Ok("modern")
            }
        })
    }

    pub(crate) fn extended_locale_expander(&self) -> Result<&LocaleExpander, DataError> {
        use super::locale::likely_subtags::*;
        self.extended_locale_expander
            .get_or_init(|| {
                use icu_provider::prelude::*;
                struct Provider {
                    common: TransformResult,
                    extended: TransformResult,
                }
                impl DataProvider<LocaleLikelySubtagsLanguageV1> for Provider {
                    fn load(
                        &self,
                        _req: DataRequest,
                    ) -> Result<DataResponse<LocaleLikelySubtagsLanguageV1>, DataError>
                    {
                        Ok(DataResponse {
                            payload: DataPayload::from_owned(self.common.as_langs()),
                            metadata: Default::default(),
                        })
                    }
                }
                impl DataProvider<LocaleLikelySubtagsScriptRegionV1> for Provider {
                    fn load(
                        &self,
                        _req: DataRequest,
                    ) -> Result<DataResponse<LocaleLikelySubtagsScriptRegionV1>, DataError>
                    {
                        Ok(DataResponse {
                            payload: DataPayload::from_owned(self.common.as_script_region()),
                            metadata: Default::default(),
                        })
                    }
                }
                impl DataProvider<LocaleLikelySubtagsExtendedV1> for Provider {
                    fn load(
                        &self,
                        _req: DataRequest,
                    ) -> Result<DataResponse<LocaleLikelySubtagsExtendedV1>, DataError>
                    {
                        Ok(DataResponse {
                            payload: DataPayload::from_owned(self.extended.as_extended()),
                            metadata: Default::default(),
                        })
                    }
                }
                let common =
                    transform(LikelySubtagsResources::try_from_cldr_cache(self)?.get_common());
                let extended =
                    transform(LikelySubtagsResources::try_from_cldr_cache(self)?.get_extended());

                LocaleExpander::try_new_extended_unstable(&Provider { common, extended }).map_err(
                    |e| {
                        DataError::custom("creating LocaleExpander in CldrCache")
                            .with_display_context(&e)
                    },
                )
            })
            .as_ref()
            .map_err(|&e| e)
    }

    /// CLDR sometimes stores locales with default scripts.
    /// Add in the likely script here to make that data reachable.
    fn add_script_extended(&self, locale: &DataLocale) -> Result<Option<DataLocale>, DataError> {
        if locale.language.is_default() || locale.script.is_some() {
            return Ok(None);
        }
        let mut new_langid =
            icu::locale::LanguageIdentifier::from((locale.language, locale.script, locale.region));
        self.extended_locale_expander()?.maximize(&mut new_langid);
        debug_assert!(
            new_langid.script.is_some(),
            "Script not found for: {new_langid:?}"
        );
        if locale.region.is_none() {
            new_langid.region = None;
        }
        Ok(Some(new_langid.into()))
    }

    /// ICU4X does not store locales with their script
    /// if the script is the default for the language.
    /// Perform that normalization mapping here.
    fn remove_script_extended(&self, locale: &DataLocale) -> Result<Option<DataLocale>, DataError> {
        if locale.language.is_default() || locale.script.is_none() {
            return Ok(None);
        }
        let mut langid =
            icu::locale::LanguageIdentifier::from((locale.language, locale.script, locale.region));
        self.extended_locale_expander()?.minimize(&mut langid);
        if langid.script.is_some() || (locale.region.is_none() && langid.region.is_some()) {
            // Wasn't able to minimize the script, or had to add a region
            return Ok(None);
        }
        // Restore the region
        langid.region = locale.region;
        Ok(Some(langid.into()))
    }
}

pub(crate) struct CldrDirNoLang<'a>(&'a CldrCache, String);

impl<'a> CldrDirNoLang<'a> {
    pub(crate) fn read_and_parse<S>(&self, file_name: &str) -> Result<&'a S, DataError>
    where
        for<'de> S: serde::Deserialize<'de> + 'static + Send + Sync,
    {
        self.0
            .serde_cache
            .read_and_parse_json(&format!("{}/{}", self.1, file_name))
    }
}

pub(crate) struct CldrDirLang<'a>(&'a CldrCache, String);

impl<'a> CldrDirLang<'a> {
    pub(crate) fn read_and_parse<S>(
        &self,
        locale: &DataLocale,
        file_name: &str,
    ) -> Result<&'a S, DataError>
    where
        for<'de> S: serde::Deserialize<'de> + 'static + Send + Sync,
    {
        let dir_suffix = self.0.dir_suffix()?;
        let path = format!("{}-{dir_suffix}/main/{locale}/{file_name}", self.1);
        if self.0.serde_cache.file_exists(&path)? {
            self.0.serde_cache.read_and_parse_json(&path)
        } else if let Some(new_locale) = self.0.add_script_extended(locale)? {
            self.read_and_parse(&new_locale, file_name)
        } else {
            Err(DataErrorKind::Io(std::io::ErrorKind::NotFound)
                .into_error()
                .with_display_context(&path))
        }
    }

    pub(crate) fn list_locales(&self) -> Result<impl Iterator<Item = DataLocale> + '_, DataError> {
        let dir_suffix = self.0.dir_suffix()?;
        let path = format!("{}-{dir_suffix}/main", self.1);
        Ok(self
            .0
            .serde_cache
            .list(&path)?
            .map(|path| -> Result<DataLocale, DataError> {
                let locale = DataLocale::from_str(&path).unwrap();
                Ok(self.0.remove_script_extended(&locale)?.unwrap_or(locale))
            })
            .collect::<Result<Vec<_>, _>>()?
            .into_iter())
    }

    pub(crate) fn file_exists(
        &self,
        lang: &DataLocale,
        file_name: &str,
    ) -> Result<bool, DataError> {
        let dir_suffix = self.0.dir_suffix()?;
        let path = format!("{}-{dir_suffix}/main/{lang}/{file_name}", self.1);
        if self.0.serde_cache.file_exists(&path)? {
            Ok(true)
        } else if let Some(new_locale) = self.0.add_script_extended(lang)? {
            self.file_exists(&new_locale, file_name)
        } else {
            Ok(false)
        }
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	#![allow(dead_code)] // features
6
7	use crate::cldr_serde::eras::EraData;
8	use crate::datetime::DatagenCalendar;
9	use crate::source::SerdeCache;
10	use crate::CoverageLevel;
11	use icu::locale::provider::{
12	LocaleLikelySubtagsExtendedV1, LocaleLikelySubtagsLanguageV1, LocaleLikelySubtagsScriptRegionV1,
13	};
14	use icu::locale::LocaleExpander;
15	use icu_provider::prelude::*;
16	use icu_provider::DataError;
17	use std::collections::BTreeMap;
18	use std::collections::HashSet;
19	use std::fmt::Debug;
20	use std::str::FromStr;
21	use std::sync::OnceLock;
22	use writeable::Writeable;
23
24	#[derive(Debug)]	×
25	pub(crate) struct CldrCache {
26	pub(crate) serde_cache: SerdeCache,
27	dir_suffix: OnceLock<Result<&'static str, DataError>>,	×
28	extended_locale_expander: OnceLock<Result<LocaleExpander, DataError>>,	×
29	#[allow(clippy::type_complexity)]
30	pub(crate) calendar_eras:	×
31	OnceLock<Result<BTreeMap<DatagenCalendar, Vec<(usize, EraData)>>, DataError>>,
32	#[cfg(feature = "experimental")]
33	// used by transforms/mod.rs
34	pub(crate) transforms: OnceLock<	×
35	Result<std::sync::Mutex<icu::experimental::transliterate::RuleCollection>, DataError>,
36	>,
37	pub(crate) tz_caches: crate::time_zones::Caches,	×
38	}
39
40	impl CldrCache {
41	pub(crate) fn from_serde_cache(serde_cache: SerdeCache) -> Self {	3✔
42	CldrCache {	3✔
43	serde_cache,	3✔
44	dir_suffix: Default::default(),	3✔
45	extended_locale_expander: Default::default(),	3✔
46	calendar_eras: Default::default(),	3✔
47	#[cfg(feature = "experimental")]
48	transforms: Default::default(),	3✔
49	tz_caches: Default::default(),	3✔
50	}	×
51	}	3✔
52
53	pub(crate) fn core(&self) -> CldrDirNoLang<'_> {	507✔
54	CldrDirNoLang(self, "cldr-core".to_owned())	507✔
55	}	507✔
56
57	pub(crate) fn numbers(&self) -> CldrDirLang<'_> {	437✔
58	CldrDirLang(self, "cldr-numbers".to_owned())	437✔
59	}	437✔
60
61	pub(crate) fn misc(&self) -> CldrDirLang<'_> {	261✔
62	CldrDirLang(self, "cldr-misc".to_owned())	261✔
63	}	261✔
64
65	pub(crate) fn bcp47(&self) -> CldrDirNoLang<'_> {	170✔
66	CldrDirNoLang(self, "cldr-bcp47/bcp47".to_string())	170✔
67	}	170✔
68
69	pub(crate) fn personnames(&self) -> CldrDirLang<'_> {	40✔
70	CldrDirLang(self, "cldr-person-names".to_owned())	40✔
71	}	40✔
72
73	pub(crate) fn displaynames(&self) -> CldrDirLang<'_> {	420✔
74	CldrDirLang(self, "cldr-localenames".to_owned())	420✔
75	}	420✔
76
77	pub(crate) fn units(&self) -> CldrDirLang<'_> {	513✔
78	CldrDirLang(self, "cldr-units".to_owned())	513✔
79	}	513✔
80
81	pub(crate) fn dates(&self, cal: &str) -> CldrDirLang<'_> {	6,748✔
82	CldrDirLang(	6,748✔
83	self,
84	if cal == "gregorian" \|\| cal == "generic" {	6,748✔
85	"cldr-dates".to_owned()	2,351✔
86	} else {
87	format!("cldr-cal-{cal}")	4,397✔
88	},
89	)
90	}	6,748✔
91
92	pub(crate) fn locales(	1✔
93	&self,
94	levels: impl IntoIterator<Item = CoverageLevel>,
95	) -> Result<Vec<DataLocale>, DataError> {
96	let levels = levels.into_iter().collect::<HashSet<_>>();	1✔
97	let mut locales: Vec<DataLocale> = self	2✔
98	.serde_cache
99	.read_and_parse_json::<crate::cldr_serde::coverage_levels::Resource>(
100	"cldr-core/coverageLevels.json",
101	)?	×
102	.coverage_levels
103	.iter()
104	.filter_map(\|(locale, c)\| levels.contains(c).then_some(locale))	167✔
105	.cloned()
106	.map(Into::into)
107	// `und` needs to be part of every set
108	.chain([Default::default()])	1✔
109	.collect();
110	locales.sort_by(\|a, b\| {	711✔
111	let b = b.write_to_string();	710✔
112	a.strict_cmp(b.as_bytes())	710✔
113	});	710✔
114	Ok(locales)	1✔
115	}	1✔
116
117	pub(crate) fn dir_suffix(&self) -> Result<&'static str, DataError> {	8,421✔
118	*self.dir_suffix.get_or_init(\|\| {	8,423✔
119	if self.serde_cache.list("cldr-misc-full")?.next().is_some() {	2✔
120	Ok("full")	2✔
121	} else {
122	Ok("modern")	×
123	}
124	})	2✔
125	}	8,421✔
126
127	pub(crate) fn extended_locale_expander(&self) -> Result<&LocaleExpander, DataError> {	331✔
128	use super::locale::likely_subtags::*;
129	self.extended_locale_expander	331✔
130	.get_or_init(\|\| {	1✔
131	use icu_provider::prelude::*;
132	struct Provider {
133	common: TransformResult,
134	extended: TransformResult,
135	}
136	impl DataProvider<LocaleLikelySubtagsLanguageV1> for Provider {
137	fn load(	1✔
138	&self,
139	_req: DataRequest,
140	) -> Result<DataResponse<LocaleLikelySubtagsLanguageV1>, DataError>
141	{
142	Ok(DataResponse {	1✔
143	payload: DataPayload::from_owned(self.common.as_langs()),	1✔
144	metadata: Default::default(),	1✔
145	})	×
146	}	1✔
147	}
148	impl DataProvider<LocaleLikelySubtagsScriptRegionV1> for Provider {
149	fn load(	1✔
150	&self,
151	_req: DataRequest,
152	) -> Result<DataResponse<LocaleLikelySubtagsScriptRegionV1>, DataError>
153	{
154	Ok(DataResponse {	1✔
155	payload: DataPayload::from_owned(self.common.as_script_region()),	1✔
156	metadata: Default::default(),	1✔
157	})	×
158	}	1✔
159	}
160	impl DataProvider<LocaleLikelySubtagsExtendedV1> for Provider {
161	fn load(	1✔
162	&self,
163	_req: DataRequest,
164	) -> Result<DataResponse<LocaleLikelySubtagsExtendedV1>, DataError>
165	{
166	Ok(DataResponse {	1✔
167	payload: DataPayload::from_owned(self.extended.as_extended()),	1✔
168	metadata: Default::default(),	1✔
169	})	×
170	}	1✔
171	}
172	let common =
173	transform(LikelySubtagsResources::try_from_cldr_cache(self)?.get_common());	1✔
174	let extended =
175	transform(LikelySubtagsResources::try_from_cldr_cache(self)?.get_extended());	1✔
176
177	LocaleExpander::try_new_extended_unstable(&Provider { common, extended }).map_err(	1✔
178	\|e\| {	×
179	DataError::custom("creating LocaleExpander in CldrCache")	×
180	.with_display_context(&e)
181	},	×
182	)
183	})	1✔
184	.as_ref()
185	.map_err(\|&e\| e)	×
186	}	331✔
187
188	/// CLDR sometimes stores locales with default scripts.
189	/// Add in the likely script here to make that data reachable.
190	fn add_script_extended(&self, locale: &DataLocale) -> Result<Option<DataLocale>, DataError> {	4✔
191	if locale.language.is_default() \|\| locale.script.is_some() {	4✔
192	return Ok(None);	2✔
193	}
194	let mut new_langid =
195	icu::locale::LanguageIdentifier::from((locale.language, locale.script, locale.region));	2✔
196	self.extended_locale_expander()?.maximize(&mut new_langid);	2✔
197	debug_assert!(	×
198	new_langid.script.is_some(),	2✔
199	"Script not found for: {new_langid:?}"
200	);
201	if locale.region.is_none() {	2✔
202	new_langid.region = None;	2✔
203	}
204	Ok(Some(new_langid.into()))	2✔
205	}	4✔
206
207	/// ICU4X does not store locales with their script
208	/// if the script is the default for the language.
209	/// Perform that normalization mapping here.
210	fn remove_script_extended(&self, locale: &DataLocale) -> Result<Option<DataLocale>, DataError> {	2,005✔
211	if locale.language.is_default() \|\| locale.script.is_none() {	2,005✔
212	return Ok(None);	1,898✔
213	}
214	let mut langid =
215	icu::locale::LanguageIdentifier::from((locale.language, locale.script, locale.region));	107✔
216	self.extended_locale_expander()?.minimize(&mut langid);	107✔
217	if langid.script.is_some() \|\| (locale.region.is_none() && langid.region.is_some()) {	107✔
218	// Wasn't able to minimize the script, or had to add a region
219	return Ok(None);	107✔
220	}
221	// Restore the region
222	langid.region = locale.region;	×
223	Ok(Some(langid.into()))	×
224	}	2,005✔
225	}
226
227	pub(crate) struct CldrDirNoLang<'a>(&'a CldrCache, String);
228
229	impl<'a> CldrDirNoLang<'a> {
230	pub(crate) fn read_and_parse<S>(&self, file_name: &str) -> Result<&'a S, DataError>	677✔
231	where
232	for<'de> S: serde::Deserialize<'de> + 'static + Send + Sync,
233	{
234	self.0	677✔
235	.serde_cache
236	.read_and_parse_json(&format!("{}/{}", self.1, file_name))	677✔
237	}	663✔
238	}
239
240	pub(crate) struct CldrDirLang<'a>(&'a CldrCache, String);
241
242	impl<'a> CldrDirLang<'a> {
243	pub(crate) fn read_and_parse<S>(	8,061✔
244	&self,
245	locale: &DataLocale,
246	file_name: &str,
247	) -> Result<&'a S, DataError>
248	where
249	for<'de> S: serde::Deserialize<'de> + 'static + Send + Sync,
250	{
251	let dir_suffix = self.0.dir_suffix()?;	8,061✔
252	let path = format!("{}-{dir_suffix}/main/{locale}/{file_name}", self.1);	8,061✔
253	if self.0.serde_cache.file_exists(&path)? {	8,061✔
254	self.0.serde_cache.read_and_parse_json(&path)	8,061✔
255	} else if let Some(new_locale) = self.0.add_script_extended(locale)? {	×
256	self.read_and_parse(&new_locale, file_name)	×
257	} else {
258	Err(DataErrorKind::Io(std::io::ErrorKind::NotFound)	×
259	.into_error()
260	.with_display_context(&path))
261	}
262	}	8,061✔
263
264	pub(crate) fn list_locales(&self) -> Result<impl Iterator<Item = DataLocale> + '_, DataError> {	110✔
265	let dir_suffix = self.0.dir_suffix()?;	110✔
266	let path = format!("{}-{dir_suffix}/main", self.1);	110✔
267	Ok(self	110✔
268	.0
269	.serde_cache
270	.list(&path)?	113✔
271	.map(\|path\| -> Result<DataLocale, DataError> {	2,116✔
272	let locale = DataLocale::from_str(&path).unwrap();	2,009✔
273	Ok(self.0.remove_script_extended(&locale)?.unwrap_or(locale))	2,005✔
274	})	2,005✔
275	.collect::<Result<Vec<_>, _>>()?	1✔
276	.into_iter())
277	}	106✔
278
279	pub(crate) fn file_exists(	253✔
280	&self,
281	lang: &DataLocale,
282	file_name: &str,
283	) -> Result<bool, DataError> {
284	let dir_suffix = self.0.dir_suffix()?;	253✔
285	let path = format!("{}-{dir_suffix}/main/{lang}/{file_name}", self.1);	253✔
286	if self.0.serde_cache.file_exists(&path)? {	253✔
287	Ok(true)	249✔
288	} else if let Some(new_locale) = self.0.add_script_extended(lang)? {	4✔
289	self.file_exists(&new_locale, file_name)	2✔
290	} else {
291	Ok(false)	2✔
292	}
293	}	253✔
294	}

zbraniecki / icu4x / 13958601093

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous