8219362155

Committed 08 Mar 2024 01:21PM UTC coverage: 75.985% (+3.0%) from 73.009%

Build # 8219362155

Build Type

push

github

Committed by

web-flow

Commit Message

Bump diplomat (#4671)

And fix some Dart renames

Run Details

49581 of 65251 relevant lines covered (75.99%)

519628.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.99

/components/locid_transform/src/fallback/mod.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! Tools for locale fallback, enabling arbitrary input locales to be mapped into the nearest
//! locale with data.

use crate::provider::*;
use icu_locid::extensions::unicode::Value;
use icu_locid::subtags::Variants;
use icu_provider::prelude::*;

#[doc(inline)]
pub use icu_provider::fallback::*;

mod algorithms;

/// Implements the algorithm defined in *[UTS #35: Locale Inheritance and Matching]*.
///
/// Note that this implementation performs some additional steps compared to the *UTS #35*
/// algorithm. See *[the design doc]* for a detailed description and [#2243](
/// https://github.com/unicode-org/icu4x/issues/2243) to track alignment with *UTS #35*.
///
/// If running fallback in a loop, use [`DataLocale::is_und()`] to break from the loop.
///
/// # Examples
///
/// ```
/// use icu_locid::locale;
/// use icu_locid_transform::fallback::LocaleFallbacker;
///
/// // Set up a LocaleFallbacker with data.
/// let fallbacker = LocaleFallbacker::new();
///
/// // Create a LocaleFallbackerIterator with a default configuration.
/// // By default, uses language priority with no additional extension keywords.
/// let mut fallback_iterator = fallbacker
///     .for_config(Default::default())
///     .fallback_for(locale!("hi-Latn-IN").into());
///
/// // Run the algorithm and check the results.
/// assert_eq!(fallback_iterator.get(), &locale!("hi-Latn-IN").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("hi-Latn").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("en-IN").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("en-001").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("en").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("und").into());
/// ```
///
/// [UTS #35: Locale Inheritance and Matching]: https://www.unicode.org/reports/tr35/#Locale_Inheritance
/// [the design doc]: https://docs.google.com/document/d/1Mp7EUyl-sFh_HZYgyeVwj88vJGpCBIWxzlCwGgLCDwM/edit
/// [language identifier]: icu_locid::LanguageIdentifier
#[doc(hidden)]
#[derive(Debug, Clone, PartialEq)]
pub struct LocaleFallbacker {
    likely_subtags: DataPayload<LocaleFallbackLikelySubtagsV1Marker>,
    parents: DataPayload<LocaleFallbackParentsV1Marker>,
    collation_supplement: Option<DataPayload<CollationFallbackSupplementV1Marker>>,
}

/// Borrowed version of [`LocaleFallbacker`].
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct LocaleFallbackerBorrowed<'a> {
    likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,
    parents: &'a LocaleFallbackParentsV1<'a>,
    collation_supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,
}

/// A [`LocaleFallbackerBorrowed`] with an associated [`LocaleFallbackConfig`].
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct LocaleFallbackerWithConfig<'a> {
    likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,
    parents: &'a LocaleFallbackParentsV1<'a>,
    supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,
    config: LocaleFallbackConfig,
}

/// Inner iteration type. Does not own the item under fallback.
#[derive(Debug)]
struct LocaleFallbackIteratorInner<'a> {
    likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,
    parents: &'a LocaleFallbackParentsV1<'a>,
    supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,
    config: LocaleFallbackConfig,
    backup_extension: Option<Value>,
    backup_subdivision: Option<Value>,
    backup_variants: Option<Variants>,
}

/// Iteration type for locale fallback operations.
///
/// Because the `Iterator` trait does not allow items to borrow from the iterator, this class does
/// not implement that trait. Instead, use `.step()` and `.get()`.
#[derive(Debug)]
pub struct LocaleFallbackIterator<'a, 'b> {
    current: DataLocale,
    inner: LocaleFallbackIteratorInner<'a>,
    phantom: core::marker::PhantomData<&'b ()>,
}

impl LocaleFallbacker {
    /// Creates a [`LocaleFallbacker`] with compiled fallback data (likely subtags and parent locales).
    ///
    /// ✨ *Enabled with the `compiled_data` Cargo feature.*
    ///
    /// [📚 Help choosing a constructor](icu_provider::constructors)
    #[cfg(feature = "compiled_data")]
    #[allow(clippy::new_ret_no_self)] // keeping constructors together
    pub const fn new<'a>() -> LocaleFallbackerBorrowed<'a> {
        let tickstatic = LocaleFallbackerBorrowed {
            likely_subtags: crate::provider::Baked::SINGLETON_FALLBACK_LIKELYSUBTAGS_V1,
            parents: crate::provider::Baked::SINGLETON_FALLBACK_PARENTS_V1,
            collation_supplement: Some(crate::provider::Baked::SINGLETON_FALLBACK_SUPPLEMENT_CO_V1),
        };
        // Safety: we're transmuting down from LocaleFallbackerBorrowed<'static> to LocaleFallbackerBorrowed<'a>
        // ZeroMaps use associated types in a way that confuse the compiler which gives up and marks them
        // as invariant. However, they are covariant, and in non-const code this covariance can be safely triggered
        // using Yokeable::transform. In const code we must transmute. In the long run we should
        // be able to `transform()` in const code, and also we will have hopefully improved map polymorphism (#3128)
        unsafe { core::mem::transmute(tickstatic) }
    }

    icu_provider::gen_any_buffer_data_constructors!(locale: skip, options: skip, error: DataError,
        #[cfg(skip)]
        functions: [
            new,
            try_new_with_any_provider,
            try_new_with_buffer_provider,
            try_new_unstable,
            Self
    ]);

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new)]
    pub fn try_new_unstable<P>(provider: &P) -> Result<Self, DataError>
    where
        P: DataProvider<LocaleFallbackLikelySubtagsV1Marker>
            + DataProvider<LocaleFallbackParentsV1Marker>
            + DataProvider<CollationFallbackSupplementV1Marker>
            + ?Sized,
    {
        let likely_subtags = provider.load(Default::default())?.take_payload()?;
        let parents = provider.load(Default::default())?.take_payload()?;
        let collation_supplement = match DataProvider::<CollationFallbackSupplementV1Marker>::load(
            provider,
            Default::default(),
        ) {
            Ok(response) => Some(response.take_payload()?),
            // It is expected that not all keys are present
            Err(DataError {
                kind: DataErrorKind::MissingDataKey,
                ..
            }) => None,
            Err(e) => return Err(e),
        };
        Ok(LocaleFallbacker {
            likely_subtags,
            parents,
            collation_supplement,
        })
    }

    /// Creates a [`LocaleFallbacker`] without fallback data. Using this constructor may result in
    /// surprising behavior, especially in multi-script languages.
    pub fn new_without_data() -> Self {
        LocaleFallbacker {
            likely_subtags: DataPayload::from_owned(Default::default()),
            parents: DataPayload::from_owned(Default::default()),
            collation_supplement: None,
        }
    }

    /// Associates a configuration with this fallbacker.
    #[inline]
    pub fn for_config(&self, config: LocaleFallbackConfig) -> LocaleFallbackerWithConfig {
        self.as_borrowed().for_config(config)
    }

    /// Derives a configuration from a [`DataKey`] and associates it
    /// with this fallbacker.
    #[inline]
    #[doc(hidden)] // will be removed in 2.0
    pub fn for_key(&self, data_key: DataKey) -> LocaleFallbackerWithConfig {
        self.for_config(data_key.fallback_config())
    }

    /// Creates a borrowed version of this fallbacker for performance.
    pub fn as_borrowed(&self) -> LocaleFallbackerBorrowed {
        LocaleFallbackerBorrowed {
            likely_subtags: self.likely_subtags.get(),
            parents: self.parents.get(),
            collation_supplement: self.collation_supplement.as_ref().map(|p| p.get()),
        }
    }
}

impl<'a> LocaleFallbackerBorrowed<'a> {
    /// Associates a configuration with this fallbacker.
    #[inline]
    pub const fn for_config(self, config: LocaleFallbackConfig) -> LocaleFallbackerWithConfig<'a> {
        LocaleFallbackerWithConfig {
            likely_subtags: self.likely_subtags,
            parents: self.parents,
            supplement: match config.fallback_supplement {
                Some(LocaleFallbackSupplement::Collation) => self.collation_supplement,
                _ => None,
            },
            config,
        }
    }
}

impl LocaleFallbackerBorrowed<'static> {
    /// Cheaply converts a `LocaleFallbackerBorrowed<'static>` into a `LocaleFallbacker`.
    pub const fn static_to_owned(self) -> LocaleFallbacker {
        LocaleFallbacker {
            likely_subtags: DataPayload::from_static_ref(self.likely_subtags),
            parents: DataPayload::from_static_ref(self.parents),
            collation_supplement: match self.collation_supplement {
                None => None,
                Some(x) => Some(DataPayload::from_static_ref(x)),
            },
        }
    }
}

impl<'a> LocaleFallbackerWithConfig<'a> {
    /// Creates an iterator based on a [`DataLocale`].
    ///
    /// If you have a [`Locale`](icu_locid::Locale), call `.into()` to get a [`DataLocale`].
    ///
    /// When first initialized, the locale is normalized according to the fallback algorithm.
    pub fn fallback_for(&self, mut locale: DataLocale) -> LocaleFallbackIterator<'a, 'static> {
        self.normalize(&mut locale);
        LocaleFallbackIterator {
            current: locale,
            inner: LocaleFallbackIteratorInner {
                likely_subtags: self.likely_subtags,
                parents: self.parents,
                supplement: self.supplement,
                config: self.config,
                backup_extension: None,
                backup_subdivision: None,
                backup_variants: None,
            },
            phantom: core::marker::PhantomData,
        }
    }
}

impl LocaleFallbackIterator<'_, '_> {
    /// Borrows the current [`DataLocale`] under fallback.
    pub fn get(&self) -> &DataLocale {
        &self.current
    }

    /// Takes the current [`DataLocale`] under fallback.
    pub fn take(self) -> DataLocale {
        self.current
    }

    /// Performs one step of the locale fallback algorithm.
    ///
    /// The fallback is completed once the inner [`DataLocale`] becomes `und`.
    pub fn step(&mut self) -> &mut Self {
        self.inner.step(&mut self.current);
        self
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	×
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	//! Tools for locale fallback, enabling arbitrary input locales to be mapped into the nearest
6	//! locale with data.
7
8	use crate::provider::*;
9	use icu_locid::extensions::unicode::Value;
10	use icu_locid::subtags::Variants;
11	use icu_provider::prelude::*;
12
13	#[doc(inline)]
14	pub use icu_provider::fallback::*;
15
16	mod algorithms;
17
18	/// Implements the algorithm defined in [UTS #35: Locale Inheritance and Matching].
19	///
20	/// Note that this implementation performs some additional steps compared to the UTS #35
21	/// algorithm. See [the design doc] for a detailed description and [#2243](
22	/// https://github.com/unicode-org/icu4x/issues/2243) to track alignment with UTS #35.
23	///
24	/// If running fallback in a loop, use [`DataLocale::is_und()`] to break from the loop.
25	///
26	/// # Examples
27	///
28	/// ```
29	/// use icu_locid::locale;
30	/// use icu_locid_transform::fallback::LocaleFallbacker;
31	///
32	/// // Set up a LocaleFallbacker with data.
33	/// let fallbacker = LocaleFallbacker::new();
34	///
35	/// // Create a LocaleFallbackerIterator with a default configuration.
36	/// // By default, uses language priority with no additional extension keywords.
37	/// let mut fallback_iterator = fallbacker
38	/// .for_config(Default::default())
39	/// .fallback_for(locale!("hi-Latn-IN").into());
40	///
41	/// // Run the algorithm and check the results.
42	/// assert_eq!(fallback_iterator.get(), &locale!("hi-Latn-IN").into());
43	/// fallback_iterator.step();
44	/// assert_eq!(fallback_iterator.get(), &locale!("hi-Latn").into());
45	/// fallback_iterator.step();
46	/// assert_eq!(fallback_iterator.get(), &locale!("en-IN").into());
47	/// fallback_iterator.step();
48	/// assert_eq!(fallback_iterator.get(), &locale!("en-001").into());
49	/// fallback_iterator.step();
50	/// assert_eq!(fallback_iterator.get(), &locale!("en").into());
51	/// fallback_iterator.step();
52	/// assert_eq!(fallback_iterator.get(), &locale!("und").into());
53	/// ```
54	///
55	/// [UTS #35: Locale Inheritance and Matching]: https://www.unicode.org/reports/tr35/#Locale_Inheritance
56	/// [the design doc]: https://docs.google.com/document/d/1Mp7EUyl-sFh_HZYgyeVwj88vJGpCBIWxzlCwGgLCDwM/edit
57	/// [language identifier]: icu_locid::LanguageIdentifier
58	#[doc(hidden)]
59	#[derive(Debug, Clone, PartialEq)]	×
60	pub struct LocaleFallbacker {
61	likely_subtags: DataPayload<LocaleFallbackLikelySubtagsV1Marker>,	×
62	parents: DataPayload<LocaleFallbackParentsV1Marker>,	×
63	collation_supplement: Option<DataPayload<CollationFallbackSupplementV1Marker>>,	×
64	}
65
66	/// Borrowed version of [`LocaleFallbacker`].
67	#[derive(Debug, Clone, Copy, PartialEq)]	×
68	pub struct LocaleFallbackerBorrowed<'a> {
69	likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,	×
70	parents: &'a LocaleFallbackParentsV1<'a>,	×
71	collation_supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,	×
72	}
73
74	/// A [`LocaleFallbackerBorrowed`] with an associated [`LocaleFallbackConfig`].
75	#[derive(Debug, Clone, Copy, PartialEq)]	×
76	pub struct LocaleFallbackerWithConfig<'a> {
77	likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,	×
78	parents: &'a LocaleFallbackParentsV1<'a>,	×
79	supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,	×
80	config: LocaleFallbackConfig,	×
81	}
82
83	/// Inner iteration type. Does not own the item under fallback.
84	#[derive(Debug)]	×
85	struct LocaleFallbackIteratorInner<'a> {
86	likely_subtags: &'a LocaleFallbackLikelySubtagsV1<'a>,	×
87	parents: &'a LocaleFallbackParentsV1<'a>,	×
88	supplement: Option<&'a LocaleFallbackSupplementV1<'a>>,	×
89	config: LocaleFallbackConfig,	×
90	backup_extension: Option<Value>,	×
91	backup_subdivision: Option<Value>,	×
92	backup_variants: Option<Variants>,	×
93	}
94
95	/// Iteration type for locale fallback operations.
96	///
97	/// Because the `Iterator` trait does not allow items to borrow from the iterator, this class does
98	/// not implement that trait. Instead, use `.step()` and `.get()`.
99	#[derive(Debug)]	×
100	pub struct LocaleFallbackIterator<'a, 'b> {
101	current: DataLocale,	×
102	inner: LocaleFallbackIteratorInner<'a>,	×
103	phantom: core::marker::PhantomData<&'b ()>,	×
104	}
105
106	impl LocaleFallbacker {
107	/// Creates a [`LocaleFallbacker`] with compiled fallback data (likely subtags and parent locales).
108	///
109	/// ✨ Enabled with the `compiled_data` Cargo feature.
110	///
111	/// [📚 Help choosing a constructor](icu_provider::constructors)
112	#[cfg(feature = "compiled_data")]
113	#[allow(clippy::new_ret_no_self)] // keeping constructors together
114	pub const fn new<'a>() -> LocaleFallbackerBorrowed<'a> {	9✔
115	let tickstatic = LocaleFallbackerBorrowed {	9✔
116	likely_subtags: crate::provider::Baked::SINGLETON_FALLBACK_LIKELYSUBTAGS_V1,
117	parents: crate::provider::Baked::SINGLETON_FALLBACK_PARENTS_V1,
118	collation_supplement: Some(crate::provider::Baked::SINGLETON_FALLBACK_SUPPLEMENT_CO_V1),	9✔
119	};
120	// Safety: we're transmuting down from LocaleFallbackerBorrowed<'static> to LocaleFallbackerBorrowed<'a>
121	// ZeroMaps use associated types in a way that confuse the compiler which gives up and marks them
122	// as invariant. However, they are covariant, and in non-const code this covariance can be safely triggered
123	// using Yokeable::transform. In const code we must transmute. In the long run we should
124	// be able to `transform()` in const code, and also we will have hopefully improved map polymorphism (#3128)
125	unsafe { core::mem::transmute(tickstatic) }	9✔
126	}	9✔
127
128	icu_provider::gen_any_buffer_data_constructors!(locale: skip, options: skip, error: DataError,
129	#[cfg(skip)]
130	functions: [
131	new,
132	try_new_with_any_provider,
133	try_new_with_buffer_provider,
134	try_new_unstable,
135	Self
136	]);
137
138	#[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new)]
139	pub fn try_new_unstable<P>(provider: &P) -> Result<Self, DataError>	7✔
140	where
141	P: DataProvider<LocaleFallbackLikelySubtagsV1Marker>
142	+ DataProvider<LocaleFallbackParentsV1Marker>
143	+ DataProvider<CollationFallbackSupplementV1Marker>
144	+ ?Sized,
145	{
146	let likely_subtags = provider.load(Default::default())?.take_payload()?;	7✔
147	let parents = provider.load(Default::default())?.take_payload()?;	7✔
148	let collation_supplement = match DataProvider::<CollationFallbackSupplementV1Marker>::load(	6✔
149	provider,
150	Default::default(),	7✔
151	) {
152	Ok(response) => Some(response.take_payload()?),	5✔
153	// It is expected that not all keys are present
154	Err(DataError {
155	kind: DataErrorKind::MissingDataKey,
156	..
157	}) => None,	×
158	Err(e) => return Err(e),	×
159	};	5✔
160	Ok(LocaleFallbacker {	5✔
161	likely_subtags,	5✔
162	parents,	5✔
163	collation_supplement,	5✔
164	})
165	}	5✔
166
167	/// Creates a [`LocaleFallbacker`] without fallback data. Using this constructor may result in
168	/// surprising behavior, especially in multi-script languages.
169	pub fn new_without_data() -> Self {	10✔
170	LocaleFallbacker {	10✔
171	likely_subtags: DataPayload::from_owned(Default::default()),	10✔
172	parents: DataPayload::from_owned(Default::default()),	10✔
173	collation_supplement: None,	10✔
174	}	×
175	}	10✔
176
177	/// Associates a configuration with this fallbacker.
178	#[inline]
179	pub fn for_config(&self, config: LocaleFallbackConfig) -> LocaleFallbackerWithConfig {	3,318✔
180	self.as_borrowed().for_config(config)	3,318✔
181	}	3,318✔
182
183	/// Derives a configuration from a [`DataKey`] and associates it
184	/// with this fallbacker.
185	#[inline]
186	#[doc(hidden)] // will be removed in 2.0
187	pub fn for_key(&self, data_key: DataKey) -> LocaleFallbackerWithConfig {	×
188	self.for_config(data_key.fallback_config())	×
189	}	×
190
191	/// Creates a borrowed version of this fallbacker for performance.
192	pub fn as_borrowed(&self) -> LocaleFallbackerBorrowed {	3,321✔
193	LocaleFallbackerBorrowed {	3,321✔
194	likely_subtags: self.likely_subtags.get(),	3,321✔
195	parents: self.parents.get(),	3,321✔
196	collation_supplement: self.collation_supplement.as_ref().map(\|p\| p.get()),	6,633✔
197	}
198	}	3,321✔
199	}
200
201	impl<'a> LocaleFallbackerBorrowed<'a> {
202	/// Associates a configuration with this fallbacker.
203	#[inline]
204	pub const fn for_config(self, config: LocaleFallbackConfig) -> LocaleFallbackerWithConfig<'a> {	3,363✔
205	LocaleFallbackerWithConfig {	3,363✔
206	likely_subtags: self.likely_subtags,	3,363✔
207	parents: self.parents,	3,363✔
208	supplement: match config.fallback_supplement {	3,363✔
209	Some(LocaleFallbackSupplement::Collation) => self.collation_supplement,	131✔
210	_ => None,	3,232✔
211	},
212	config,	3,363✔
213	}
214	}	3,363✔
215	}
216
217	impl LocaleFallbackerBorrowed<'static> {
218	/// Cheaply converts a `LocaleFallbackerBorrowed<'static>` into a `LocaleFallbacker`.
219	pub const fn static_to_owned(self) -> LocaleFallbacker {	2✔
220	LocaleFallbacker {	2✔
221	likely_subtags: DataPayload::from_static_ref(self.likely_subtags),	2✔
222	parents: DataPayload::from_static_ref(self.parents),	2✔
223	collation_supplement: match self.collation_supplement {	2✔
224	None => None,	×
225	Some(x) => Some(DataPayload::from_static_ref(x)),	2✔
226	},
227	}	×
228	}	2✔
229	}
230
231	impl<'a> LocaleFallbackerWithConfig<'a> {
232	/// Creates an iterator based on a [`DataLocale`].
233	///
234	/// If you have a [`Locale`](icu_locid::Locale), call `.into()` to get a [`DataLocale`].
235	///
236	/// When first initialized, the locale is normalized according to the fallback algorithm.
237	pub fn fallback_for(&self, mut locale: DataLocale) -> LocaleFallbackIterator<'a, 'static> {	5,113✔
238	self.normalize(&mut locale);	5,113✔
239	LocaleFallbackIterator {	5,113✔
240	current: locale,	5,113✔
241	inner: LocaleFallbackIteratorInner {	5,113✔
242	likely_subtags: self.likely_subtags,	5,113✔
243	parents: self.parents,	5,113✔
244	supplement: self.supplement,	5,113✔
245	config: self.config,	5,113✔
246	backup_extension: None,	5,113✔
247	backup_subdivision: None,	5,113✔
248	backup_variants: None,	5,113✔
249	},
250	phantom: core::marker::PhantomData,
251	}
252	}	5,113✔
253	}
254
255	impl LocaleFallbackIterator<'_, '_> {
256	/// Borrows the current [`DataLocale`] under fallback.
257	pub fn get(&self) -> &DataLocale {	32,931✔
258	&self.current
259	}	32,931✔
260
261	/// Takes the current [`DataLocale`] under fallback.
262	pub fn take(self) -> DataLocale {	1,258✔
263	self.current	1,258✔
264	}	1,258✔
265
266	/// Performs one step of the locale fallback algorithm.
267	///
268	/// The fallback is completed once the inner [`DataLocale`] becomes `und`.
269	pub fn step(&mut self) -> &mut Self {	5,755✔
270	self.inner.step(&mut self.current);	5,755✔
271	self
272	}	5,755✔
273	}

zbraniecki / icu4x / 8219362155

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous