13958601093

Committed 19 Mar 2025 04:17PM UTC coverage: 74.164% (-1.5%) from 75.71%

Build # 13958601093

Build Type

push

github

Committed by

web-flow

Commit Message

Clean up properties docs (#6315)

Run Details

58056 of 78281 relevant lines covered (74.16%)

819371.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.3

/provider/source/src/transforms/mod.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use super::cldr_serde::transforms;
use super::CldrCache;
use crate::SourceDataProvider;
use icu::experimental::transliterate::provider::*;
use icu::experimental::transliterate::RuleCollection;
use icu::locale::Locale;
use icu_provider::prelude::*;
use itertools::Itertools;
use std::collections::HashSet;
use std::sync::Mutex;

impl CldrCache {
    pub(crate) fn transforms(&self) -> Result<&Mutex<RuleCollection>, DataError> {
        self.transforms
            .get_or_init(|| {
                fn invent_bcp47(transform: &str) -> Locale {
                    let transform = transform.to_ascii_lowercase();
                    let r = ["und", "x"]
                        .into_iter()
                        .chain(
                            transform
                                .split('-')
                                .map(|t| &t[..core::cmp::min(8, t.len())]),
                        )
                        .join("-")
                        .parse()
                        .unwrap();
                    r
                }

                let mut provider = RuleCollection::default();

                for transform in self.serde_cache.list("cldr-transforms/transforms/")? {
                    let Some(transform) = transform.strip_suffix(".json") else {
                        continue;
                    };

                    let metadata = self
                        .serde_cache
                        .read_and_parse_json::<transforms::Resource>(&format!(
                            "cldr-transforms/transforms/{transform}.json"
                        ))?;
                    let source = self.serde_cache.root.read_to_string(&format!(
                        "cldr-transforms/transforms/{}",
                        metadata.rules_file
                    ))?;

                    // Unimplemented built-in transliterators
                    if source.contains("Any-BreakInternal") || source.contains("Any-Title") {
                        continue;
                    }

                    if matches!(
                        metadata.direction,
                        transforms::Direction::Forward | transforms::Direction::Both
                    ) {
                        let bcp47_alias =
                            if let Some(bcp47_aliases) = metadata.alias_bcp47.as_ref() {
                                Locale::try_from_str(bcp47_aliases.split(' ').next().unwrap())
                                    .map_err(|_| {
                                        DataError::custom("invalid locale")
                                            .with_display_context(bcp47_aliases)
                                    })?
                            } else {
                                invent_bcp47(transform)
                            };

                        provider.register_source(
                            &bcp47_alias,
                            source.clone(),
                            metadata
                                .alias
                                .as_deref()
                                .into_iter()
                                .chain(
                                    (|| {
                                        Some(match metadata.variant.as_deref() {
                                            None => {
                                                format!(
                                                    "{}-{}",
                                                    metadata.source.as_deref()?,
                                                    metadata.target.as_deref()?
                                                )
                                            }
                                            Some(v) => {
                                                format!(
                                                    "{}-{}/{v}",
                                                    metadata.source.as_deref()?,
                                                    metadata.target.as_deref()?
                                                )
                                            }
                                        })
                                    })()
                                    .as_deref(),
                                )
                                .chain(
                                    metadata
                                        .alias_bcp47
                                        .as_deref()
                                        .unwrap_or_default()
                                        .split(' ')
                                        .skip(1),
                                )
                                .chain(
                                    metadata
                                        .alias
                                        .as_deref()
                                        .unwrap_or_default()
                                        .split(' ')
                                        .skip(1),
                                ),
                            false,
                            metadata.visibility == transforms::Visibility::External,
                        );
                    }

                    if matches!(
                        metadata.direction,
                        transforms::Direction::Backward | transforms::Direction::Both
                    ) {
                        let bcp47_alias =
                            if let Some(bcp47_aliases) = metadata.backward_alias_bcp47.as_ref() {
                                Locale::try_from_str(bcp47_aliases.split(' ').next().unwrap())
                                    .map_err(|_| {
                                        DataError::custom("invalid locale")
                                            .with_display_context(bcp47_aliases)
                                    })?
                            } else {
                                invent_bcp47(&format!("{transform}-rev"))
                            };
                        provider.register_source(
                            &bcp47_alias,
                            source,
                            metadata
                                .backward_alias
                                .as_deref()
                                .into_iter()
                                .chain(
                                    (|| {
                                        Some(match metadata.variant.as_deref() {
                                            None => {
                                                format!(
                                                    "{}-{}",
                                                    metadata.target.as_deref()?,
                                                    metadata.source.as_deref()?
                                                )
                                            }
                                            Some(v) => {
                                                format!(
                                                    "{}-{}/{v}",
                                                    metadata.target.as_deref()?,
                                                    metadata.source.as_deref()?
                                                )
                                            }
                                        })
                                    })()
                                    .as_deref(),
                                )
                                .chain(
                                    metadata
                                        .backward_alias_bcp47
                                        .as_deref()
                                        .unwrap_or_default()
                                        .split(' ')
                                        .skip(1),
                                ),
                            true,
                            metadata.visibility == transforms::Visibility::External,
                        );
                    }
                }
                Ok(Mutex::new(provider))
            })
            .as_ref()
            .map_err(|&e| e)
    }
}

impl DataProvider<TransliteratorRulesV1> for SourceDataProvider {
    fn load(&self, req: DataRequest) -> Result<DataResponse<TransliteratorRulesV1>, DataError> {
        self.check_req::<TransliteratorRulesV1>(req)?;
        self.cldr()?
            .transforms()?
            .lock()
            .expect("poison")
            .as_provider_unstable(self, self, self)?
            .load(req)
    }
}

impl crate::IterableDataProviderCached<TransliteratorRulesV1> for SourceDataProvider {
    fn iter_ids_cached(&self) -> Result<HashSet<DataIdentifierCow<'static>>, DataError> {
        Ok(self
            .cldr()?
            .transforms()?
            .lock()
            .expect("poison")
            .as_provider_unstable(self, self, self)?
            .iter_ids()?
            .into_iter()
            .map(|id| id.as_borrowed().into_owned())
            .collect())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_de_ascii_forward() {
        let provider = SourceDataProvider::new_testing();

        let _data: DataPayload<TransliteratorRulesV1> = provider
            .load(DataRequest {
                id: DataIdentifierCow::from_marker_attributes(
                    DataMarkerAttributes::from_str_or_panic("de-t-de-d0-ascii"),
                )
                .as_borrowed(),
                ..Default::default()
            })
            .unwrap()
            .payload;
    }

    #[test]
    fn test_latin_ascii_backward() {
        let provider = SourceDataProvider::new_testing();

        let _data: DataPayload<TransliteratorRulesV1> = provider
            .load(DataRequest {
                id: DataIdentifierCow::from_marker_attributes(
                    DataMarkerAttributes::from_str_or_panic("und-latn-t-s0-ascii"),
                )
                .as_borrowed(),
                ..Default::default()
            })
            .unwrap()
            .payload;
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	use super::cldr_serde::transforms;
6	use super::CldrCache;
7	use crate::SourceDataProvider;
8	use icu::experimental::transliterate::provider::*;
9	use icu::experimental::transliterate::RuleCollection;
10	use icu::locale::Locale;
11	use icu_provider::prelude::*;
12	use itertools::Itertools;
13	use std::collections::HashSet;
14	use std::sync::Mutex;
15
16	impl CldrCache {
17	pub(crate) fn transforms(&self) -> Result<&Mutex<RuleCollection>, DataError> {	12✔
18	self.transforms	12✔
19	.get_or_init(\|\| {	1✔
20	fn invent_bcp47(transform: &str) -> Locale {	2✔
21	let transform = transform.to_ascii_lowercase();	2✔
22	let r = ["und", "x"]	4✔
23	.into_iter()	2✔
24	.chain(
25	transform	2✔
26	.split('-')
27	.map(\|t\| &t[..core::cmp::min(8, t.len())]),	4✔
28	)	2✔
29	.join("-")
30	.parse()
31	.unwrap();	2✔
32	r	2✔
33	}	2✔
34
35	let mut provider = RuleCollection::default();	1✔
36
37	for transform in self.serde_cache.list("cldr-transforms/transforms/")? {	1✔
38	let Some(transform) = transform.strip_suffix(".json") else {	14✔
39	continue;
40	};
41
42	let metadata = self	14✔
43	.serde_cache
44	.read_and_parse_json::<transforms::Resource>(&format!(	7✔
45	"cldr-transforms/transforms/{transform}.json"
46	))?;	7✔
47	let source = self.serde_cache.root.read_to_string(&format!(	7✔
48	"cldr-transforms/transforms/{}",
49	metadata.rules_file
50	))?;	7✔
51
52	// Unimplemented built-in transliterators
53	if source.contains("Any-BreakInternal") \|\| source.contains("Any-Title") {	7✔
54	continue;
55	}
56
57	if matches!(	7✔
58	metadata.direction,	7✔
59	transforms::Direction::Forward \| transforms::Direction::Both
60	) {
61	let bcp47_alias =
62	if let Some(bcp47_aliases) = metadata.alias_bcp47.as_ref() {	12✔
63	Locale::try_from_str(bcp47_aliases.split(' ').next().unwrap())	5✔
64	.map_err(\|_\| {	×
65	DataError::custom("invalid locale")	×
66	.with_display_context(bcp47_aliases)	×
67	})?	×
68	} else {
69	invent_bcp47(transform)	2✔
70	};
71
72	provider.register_source(	7✔
73	&bcp47_alias,
74	source.clone(),	7✔
75	metadata	28✔
76	.alias
77	.as_deref()
78	.into_iter()
79	.chain(
80	(\|\| {	14✔
81	Some(match metadata.variant.as_deref() {	14✔
82	None => {
83	format!(	6✔
84	"{}-{}",
85	metadata.source.as_deref()?,	6✔
86	metadata.target.as_deref()?	6✔
87	)
88	}
89	Some(v) => {	1✔
90	format!(	1✔
91	"{}-{}/{v}",
92	metadata.source.as_deref()?,	1✔
93	metadata.target.as_deref()?	8✔
94	)
95	}
96	})
97	})()	7✔
98	.as_deref(),
99	)
100	.chain(
101	metadata	7✔
102	.alias_bcp47
103	.as_deref()
104	.unwrap_or_default()
105	.split(' ')
106	.skip(1),
107	)
108	.chain(
109	metadata	7✔
110	.alias
111	.as_deref()
112	.unwrap_or_default()
113	.split(' ')
114	.skip(1),
115	),
116	false,
117	metadata.visibility == transforms::Visibility::External,	7✔
118	);	7✔
119	}	7✔
120
121	if matches!(	7✔
122	metadata.direction,	7✔
123	transforms::Direction::Backward \| transforms::Direction::Both
124	) {
125	let bcp47_alias =
126	if let Some(bcp47_aliases) = metadata.backward_alias_bcp47.as_ref() {	4✔
127	Locale::try_from_str(bcp47_aliases.split(' ').next().unwrap())	2✔
128	.map_err(\|_\| {	×
129	DataError::custom("invalid locale")	×
130	.with_display_context(bcp47_aliases)	×
131	})?	×
132	} else {
133	invent_bcp47(&format!("{transform}-rev"))	×
134	};	×
135	provider.register_source(	2✔
136	&bcp47_alias,
137	source,	2✔
138	metadata	6✔
139	.backward_alias
140	.as_deref()
141	.into_iter()
142	.chain(
143	(\|\| {	4✔
144	Some(match metadata.variant.as_deref() {	4✔
145	None => {
146	format!(	2✔
147	"{}-{}",
148	metadata.target.as_deref()?,	2✔
149	metadata.source.as_deref()?	2✔
150	)
151	}
152	Some(v) => {	×
153	format!(	×
154	"{}-{}/{v}",
155	metadata.target.as_deref()?,	×
156	metadata.source.as_deref()?	2✔
157	)
158	}
159	})
160	})()	2✔
161	.as_deref(),
162	)
163	.chain(
164	metadata	2✔
165	.backward_alias_bcp47
166	.as_deref()
167	.unwrap_or_default()
168	.split(' ')
169	.skip(1),
170	),
171	true,
172	metadata.visibility == transforms::Visibility::External,	2✔
173	);	2✔
174	}	2✔
175	}	15✔
176	Ok(Mutex::new(provider))	1✔
177	})	29✔
178	.as_ref()
179	.map_err(\|&e\| e)	×
180	}	12✔
181	}
182
183	impl DataProvider<TransliteratorRulesV1> for SourceDataProvider {
184	fn load(&self, req: DataRequest) -> Result<DataResponse<TransliteratorRulesV1>, DataError> {	11✔
185	self.check_req::<TransliteratorRulesV1>(req)?;	11✔
186	self.cldr()?	11✔
187	.transforms()?	×
188	.lock()
189	.expect("poison")
190	.as_provider_unstable(self, self, self)?	×
191	.load(req)
192	}	11✔
193	}
194
195	impl crate::IterableDataProviderCached<TransliteratorRulesV1> for SourceDataProvider {
196	fn iter_ids_cached(&self) -> Result<HashSet<DataIdentifierCow<'static>>, DataError> {	1✔
197	Ok(self	1✔
198	.cldr()?	×
199	.transforms()?	×
200	.lock()
201	.expect("poison")
202	.as_provider_unstable(self, self, self)?	×
203	.iter_ids()?	×
204	.into_iter()
205	.map(\|id\| id.as_borrowed().into_owned())	9✔
206	.collect())
207	}	1✔
208	}
209
210	#[cfg(test)]
211	mod tests {
212	use super::*;
213
214	#[test]
215	fn test_de_ascii_forward() {	2✔
216	let provider = SourceDataProvider::new_testing();	1✔
217
218	let _data: DataPayload<TransliteratorRulesV1> = provider	1✔
219	.load(DataRequest {	1✔
220	id: DataIdentifierCow::from_marker_attributes(	1✔
221	DataMarkerAttributes::from_str_or_panic("de-t-de-d0-ascii"),	1✔
222	)
223	.as_borrowed(),
224	..Default::default()	1✔
225	})
226	.unwrap()
227	.payload;	1✔
228	}	2✔
229
230	#[test]
231	fn test_latin_ascii_backward() {	2✔
232	let provider = SourceDataProvider::new_testing();	1✔
233
234	let _data: DataPayload<TransliteratorRulesV1> = provider	1✔
235	.load(DataRequest {	1✔
236	id: DataIdentifierCow::from_marker_attributes(	1✔
237	DataMarkerAttributes::from_str_or_panic("und-latn-t-s0-ascii"),	1✔
238	)
239	.as_borrowed(),
240	..Default::default()	1✔
241	})
242	.unwrap()
243	.payload;	1✔
244	}	2✔
245	}

zbraniecki / icu4x / 13958601093

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous