6815798908

Committed 09 Nov 2023 05:17PM UTC coverage: 72.607% (-2.4%) from 75.01%

Build # 6815798908

Build Type

push

github

Committed by

web-flow

Commit Message

Implement `Any/BufferProvider` for some smart pointers (#4255)

Allows storing them as a `Box<dyn Any/BufferProvider>` without using a
wrapper type that implements the trait.

Run Details

44281 of 60987 relevant lines covered (72.61%)

201375.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.58

/components/locid_transform/src/canonicalizer.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! The collection of code for locale canonicalization.

use crate::provider::*;
use crate::LocaleTransformError;
use alloc::vec::Vec;
use core::cmp::Ordering;

use crate::LocaleExpander;
use crate::TransformResult;
use icu_locid::subtags::{Language, Region, Script};
use icu_locid::{
    extensions::unicode::key,
    subtags::{language, Variant, Variants},
    LanguageIdentifier, Locale,
};
use icu_provider::prelude::*;
use tinystr::TinyAsciiStr;

/// Implements the algorithm defined in *[UTS #35: Annex C, LocaleId Canonicalization]*.
///
/// # Examples
///
/// ```
/// use icu_locid::Locale;
/// use icu_locid_transform::{LocaleCanonicalizer, TransformResult};
///
/// let lc = LocaleCanonicalizer::new();
///
/// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap();
/// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified);
/// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap());
/// ```
///
/// [UTS #35: Annex C, LocaleId Canonicalization]: http://unicode.org/reports/tr35/#LocaleId_Canonicalization
#[derive(Debug)]
pub struct LocaleCanonicalizer {
    /// Data to support canonicalization.
    aliases: DataPayload<AliasesV1Marker>,
    /// Likely subtags implementation for delegation.
    expander: LocaleExpander,
}

#[inline]
fn uts35_rule_matches<'a, I>(
    source: &Locale,
    language: Language,
    script: Option<Script>,
    region: Option<Region>,
    raw_variants: I,
) -> bool
where
    I: Iterator<Item = &'a str>,
{
    (language.is_empty() || language == source.id.language)
        && (script.is_none() || script == source.id.script)
        && (region.is_none() || region == source.id.region)
        && {
            // Checks if variants are a subset of source variants.
            // As both iterators are sorted, this can be done linearly.
            let mut source_variants = source.id.variants.iter();
            'outer: for it in raw_variants {
                for cand in source_variants.by_ref() {
                    match cand.strict_cmp(it.as_bytes()) {
                        Ordering::Equal => {
                            continue 'outer;
                        }
                        Ordering::Less => {}
                        _ => {
                            return false;
                        }
                    }
                }
                return false;
            }
            true
        }
}

fn uts35_replacement<'a, I>(
    source: &mut Locale,
    ruletype_has_language: bool,
    ruletype_has_script: bool,
    ruletype_has_region: bool,
    ruletype_variants: Option<I>,
    replacement: &LanguageIdentifier,
) where
    I: Iterator<Item = &'a str>,
{
    if ruletype_has_language || (source.id.language.is_empty() && !replacement.language.is_empty())
    {
        source.id.language = replacement.language;
    }
    if ruletype_has_script || (source.id.script.is_none() && replacement.script.is_some()) {
        source.id.script = replacement.script;
    }
    if ruletype_has_region || (source.id.region.is_none() && replacement.region.is_some()) {
        source.id.region = replacement.region;
    }
    if let Some(skips) = ruletype_variants {
        // The rule matches if the ruletype variants are a subset of the source variants.
        // This means ja-Latn-fonipa-hepburn-heploc matches against the rule for
        // hepburn-heploc and is canonicalized to ja-Latn-alalc97-fonipa

        // We're merging three sorted deduped iterators into a new sequence:
        // sources - skips + replacements

        let mut sources = source.id.variants.iter().copied().peekable();
        let mut replacements = replacement.variants.iter().copied().peekable();
        let mut skips = skips.peekable();

        let mut variants: Vec<Variant> = Vec::new();

        loop {
            match (sources.peek(), skips.peek(), replacements.peek()) {
                (Some(&source), Some(skip), _)
                    if source.strict_cmp(skip.as_bytes()) == Ordering::Greater =>
                {
                    skips.next();
                }
                (Some(&source), Some(skip), _)
                    if source.strict_cmp(skip.as_bytes()) == Ordering::Equal =>
                {
                    skips.next();
                    sources.next();
                }
                (Some(&source), _, Some(&replacement))
                    if replacement.cmp(&source) == Ordering::Less =>
                {
                    variants.push(replacement);
                    replacements.next();
                }
                (Some(&source), _, Some(&replacement))
                    if replacement.cmp(&source) == Ordering::Equal =>
                {
                    variants.push(source);
                    sources.next();
                    replacements.next();
                }
                (Some(&source), _, _) => {
                    variants.push(source);
                    sources.next();
                }
                (None, _, Some(&replacement)) => {
                    variants.push(replacement);
                    replacements.next();
                }
                (None, _, None) => {
                    break;
                }
            }
        }
        source.id.variants = Variants::from_vec_unchecked(variants);
    }
}

#[inline]
fn uts35_check_language_rules(
    locale: &mut Locale,
    alias_data: &DataPayload<AliasesV1Marker>,
) -> TransformResult {
    if !locale.id.language.is_empty() {
        let lang: TinyAsciiStr<3> = locale.id.language.into();
        let replacement = if lang.len() == 2 {
            alias_data
                .get()
                .language_len2
                .get(&lang.resize().to_unvalidated())
        } else {
            alias_data.get().language_len3.get(&lang.to_unvalidated())
        };

        if let Some(replacement) = replacement {
            if let Ok(langid) = replacement.parse() {
                uts35_replacement::<core::iter::Empty<&str>>(
                    locale, true, false, false, None, &langid,
                );
                return TransformResult::Modified;
            }
        }
    }

    TransformResult::Unmodified
}

fn is_iter_sorted<I, T>(mut iter: I) -> bool
where
    I: Iterator<Item = T>,
    T: PartialOrd,
{
    if let Some(mut last) = iter.next() {
        for curr in iter {
            if last > curr {
                return false;
            }
            last = curr;
        }
    }
    true
}

#[cfg(feature = "compiled_data")]
impl Default for LocaleCanonicalizer {
    fn default() -> Self {
        Self::new()
    }
}

impl LocaleCanonicalizer {
    /// A constructor which creates a [`LocaleCanonicalizer`] from compiled data.
    ///
    /// ✨ *Enabled with the `compiled_data` Cargo feature.*
    ///
    /// [📚 Help choosing a constructor](icu_provider::constructors)
    #[cfg(feature = "compiled_data")]
    pub const fn new() -> Self {
        Self::new_with_expander(LocaleExpander::new_extended())
    }

    // Note: This is a custom impl because the bounds on LocaleExpander::try_new_unstable changed
    #[doc = icu_provider::gen_any_buffer_unstable_docs!(ANY, Self::new)]
    pub fn try_new_with_any_provider(
        provider: &(impl AnyProvider + ?Sized),
    ) -> Result<LocaleCanonicalizer, LocaleTransformError> {
        let expander = LocaleExpander::try_new_with_any_provider(provider)?;
        Self::try_new_with_expander_unstable(&provider.as_downcasting(), expander)
    }

    // Note: This is a custom impl because the bounds on LocaleExpander::try_new_unstable changed
    #[doc = icu_provider::gen_any_buffer_unstable_docs!(BUFFER, Self::new)]
    #[cfg(feature = "serde")]
    pub fn try_new_with_buffer_provider(
        provider: &(impl BufferProvider + ?Sized),
    ) -> Result<LocaleCanonicalizer, LocaleTransformError> {
        let expander = LocaleExpander::try_new_with_buffer_provider(provider)?;
        Self::try_new_with_expander_unstable(&provider.as_deserializing(), expander)
    }

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new)]
    pub fn try_new_unstable<P>(provider: &P) -> Result<LocaleCanonicalizer, LocaleTransformError>
    where
        P: DataProvider<AliasesV1Marker>
            + DataProvider<LikelySubtagsForLanguageV1Marker>
            + DataProvider<LikelySubtagsForScriptRegionV1Marker>
            + ?Sized,
    {
        let expander = LocaleExpander::try_new_unstable(provider)?;
        Self::try_new_with_expander_unstable(provider, expander)
    }

    /// Creates a [`LocaleCanonicalizer`] with a custom [`LocaleExpander`] and compiled data.
    ///
    /// ✨ *Enabled with the `compiled_data` Cargo feature.*
    ///
    /// [📚 Help choosing a constructor](icu_provider::constructors)
    #[cfg(feature = "compiled_data")]
    pub const fn new_with_expander(expander: LocaleExpander) -> Self {
        Self {
            aliases: DataPayload::from_static_ref(
                crate::provider::Baked::SINGLETON_LOCID_TRANSFORM_ALIASES_V1,
            ),
            expander,
        }
    }

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_with_expander)]
    pub fn try_new_with_expander_unstable<P>(
        provider: &P,
        expander: LocaleExpander,
    ) -> Result<LocaleCanonicalizer, LocaleTransformError>
    where
        P: DataProvider<AliasesV1Marker> + ?Sized,
    {
        let aliases: DataPayload<AliasesV1Marker> =
            provider.load(Default::default())?.take_payload()?;

        Ok(LocaleCanonicalizer { aliases, expander })
    }

    icu_provider::gen_any_buffer_data_constructors!(
        locale: skip,
        options: LocaleExpander,
        error: LocaleTransformError,
        #[cfg(skip)]
        functions: [
            new_with_expander,
            try_new_with_expander_with_any_provider,
            try_new_with_expander_with_buffer_provider,
            try_new_with_expander_unstable,
            Self,
        ]
    );

    /// The canonicalize method potentially updates a passed in locale in place
    /// depending up the results of running the canonicalization algorithm
    /// from <http://unicode.org/reports/tr35/#LocaleId_Canonicalization>.
    ///
    /// Some BCP47 canonicalization data is not part of the CLDR json package. Because
    /// of this, some canonicalizations are not performed, e.g. the canonicalization of
    /// `und-u-ca-islamicc` to `und-u-ca-islamic-civil`. This will be fixed in a future
    /// release once the missing data has been added to the CLDR json data. See:
    /// <https://github.com/unicode-org/icu4x/issues/746>
    ///
    /// # Examples
    ///
    /// ```
    /// use icu_locid::Locale;
    /// use icu_locid_transform::{LocaleCanonicalizer, TransformResult};
    ///
    /// let lc = LocaleCanonicalizer::new();
    ///
    /// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap();
    /// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified);
    /// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap());
    /// ```
    pub fn canonicalize(&self, locale: &mut Locale) -> TransformResult {
        let mut result = TransformResult::Unmodified;

        // This loops until we get a 'fixed point', where applying the rules do not
        // result in any more changes.
        'outer: loop {
            // These are linear searches due to the ordering imposed by the canonicalization
            // rules, where rules with more variants should be considered first. With the
            // current data in CLDR, we will only do this for locales which have variants,
            // or new rules which we haven't special-cased yet (of which there are fewer
            // than 20).
            if !locale.id.variants.is_empty() {
                // These language/variant comibnations have around 20 rules
                for StrStrPair(raw_lang_variants, raw_to) in self
                    .aliases
                    .get()
                    .language_variants
                    .iter()
                    .map(zerofrom::ZeroFrom::zero_from)
                {
                    let (raw_lang, raw_variants) = {
                        let mut subtags = raw_lang_variants.split('-');
                        (
                            // str::split can't return empty iterators
                            unsafe { subtags.next().unwrap_unchecked() },
                            subtags,
                        )
                    };
                    if is_iter_sorted(raw_variants.clone()) {
                        if let Ok(lang) = raw_lang.parse::<Language>() {
                            if uts35_rule_matches(locale, lang, None, None, raw_variants.clone()) {
                                if let Ok(to) = raw_to.parse() {
                                    uts35_replacement(
                                        locale,
                                        !lang.is_empty(),
                                        false,
                                        false,
                                        Some(raw_variants),
                                        &to,
                                    );
                                    result = TransformResult::Modified;
                                    continue 'outer;
                                }
                            }
                        }
                    }
                }
            } else {
                // These are absolute fallbacks, and currently empty.
                for StrStrPair(raw_from, raw_to) in self
                    .aliases
                    .get()
                    .language
                    .iter()
                    .map(zerofrom::ZeroFrom::zero_from)
                {
                    if let Ok(from) = raw_from.parse::<LanguageIdentifier>() {
                        if uts35_rule_matches(
                            locale,
                            from.language,
                            from.script,
                            from.region,
                            from.variants.iter().map(Variant::as_str),
                        ) {
                            if let Ok(to) = raw_to.parse() {
                                uts35_replacement(
                                    locale,
                                    !from.language.is_empty(),
                                    from.script.is_some(),
                                    from.region.is_some(),
                                    Some(from.variants.iter().map(Variant::as_str)),
                                    &to,
                                );
                                result = TransformResult::Modified;
                                continue 'outer;
                            }
                        }
                    }
                }
            }

            if !locale.id.language.is_empty() {
                // If the region is specified, check sgn-region rules first
                if let Some(region) = locale.id.region {
                    if locale.id.language == language!("sgn") {
                        if let Some(&sgn_lang) = self
                            .aliases
                            .get()
                            .sgn_region
                            .get(&region.into_tinystr().to_unvalidated())
                        {
                            uts35_replacement::<core::iter::Empty<&str>>(
                                locale,
                                true,
                                false,
                                true,
                                None,
                                &sgn_lang.into(),
                            );
                            result = TransformResult::Modified;
                            continue;
                        }
                    }
                }

                if uts35_check_language_rules(locale, &self.aliases) == TransformResult::Modified {
                    result = TransformResult::Modified;
                    continue;
                }
            }

            if let Some(script) = locale.id.script {
                if let Some(&replacement) = self
                    .aliases
                    .get()
                    .script
                    .get(&script.into_tinystr().to_unvalidated())
                {
                    locale.id.script = Some(replacement);
                    result = TransformResult::Modified;
                    continue;
                }
            }

            if let Some(region) = locale.id.region {
                let replacement = if region.is_alphabetic() {
                    self.aliases
                        .get()
                        .region_alpha
                        .get(&region.into_tinystr().resize().to_unvalidated())
                } else {
                    self.aliases
                        .get()
                        .region_num
                        .get(&region.into_tinystr().to_unvalidated())
                };
                if let Some(&replacement) = replacement {
                    locale.id.region = Some(replacement);
                    result = TransformResult::Modified;
                    continue;
                }

                if let Some(regions) = self
                    .aliases
                    .get()
                    .complex_region
                    .get(&region.into_tinystr().to_unvalidated())
                {
                    // Skip if regions are empty
                    if let Some(default_region) = regions.get(0) {
                        let mut maximized = LanguageIdentifier {
                            language: locale.id.language,
                            script: locale.id.script,
                            region: None,
                            variants: Variants::default(),
                        };

                        locale.id.region = Some(
                            match (self.expander.maximize(&mut maximized), maximized.region) {
                                (TransformResult::Modified, Some(candidate))
                                    if regions.iter().any(|x| x == candidate) =>
                                {
                                    candidate
                                }
                                _ => default_region,
                            },
                        );
                        result = TransformResult::Modified;
                        continue;
                    }
                }
            }

            if !locale.id.variants.is_empty() {
                let mut modified = Vec::new();
                let mut unmodified = Vec::new();
                for &variant in locale.id.variants.iter() {
                    if let Some(&updated) = self
                        .aliases
                        .get()
                        .variant
                        .get(&variant.into_tinystr().to_unvalidated())
                    {
                        modified.push(updated);
                    } else {
                        unmodified.push(variant);
                    }
                }

                if !modified.is_empty() {
                    modified.extend(unmodified);
                    modified.sort();
                    modified.dedup();
                    locale.id.variants = Variants::from_vec_unchecked(modified);
                    result = TransformResult::Modified;
                    continue;
                }
            }

            // Nothing matched in this iteration, we're done.
            break;
        }

        // Handle Locale extensions in their own loops, because these rules do not interact
        // with each other.
        if let Some(lang) = &locale.extensions.transform.lang {
            let mut tlang: Locale = lang.clone().into();
            let mut matched = false;
            loop {
                if uts35_check_language_rules(&mut tlang, &self.aliases)
                    == TransformResult::Modified
                {
                    result = TransformResult::Modified;
                    matched = true;
                    continue;
                }

                break;
            }

            if matched {
                locale.extensions.transform.lang = Some(tlang.id);
            }
        }

        // The `rg` region override and `sd` regional subdivision keys may contain
        // language codes that require canonicalization.
        for key in &[key!("rg"), key!("sd")] {
            if let Some(value) = locale.extensions.unicode.keywords.get_mut(key) {
                if let &[only_value] = value.as_tinystr_slice() {
                    if let Some(modified_value) = self
                        .aliases
                        .get()
                        .subdivision
                        .get(&only_value.resize().to_unvalidated())
                    {
                        if let Ok(modified_value) = modified_value.parse() {
                            *value = modified_value;
                            result = TransformResult::Modified;
                        }
                    }
                }
            }
        }

        result
    }
}

#[test]
fn test_uts35_rule_matches() {
    for (source, rule, result) in [
        ("ja", "und", true),
        ("und-heploc-hepburn", "und-hepburn", true),
        ("ja-heploc-hepburn", "und-hepburn", true),
        ("ja-hepburn", "und-hepburn-heploc", false),
    ] {
        let source = source.parse().unwrap();
        let rule = rule.parse::<LanguageIdentifier>().unwrap();
        assert_eq!(
            uts35_rule_matches(
                &source,
                rule.language,
                rule.script,
                rule.region,
                rule.variants.iter().map(Variant::as_str),
            ),
            result,
            "{source}"
        );
    }
}

#[test]
fn test_uts35_replacement() {
    for (locale, rule_0, rule_1, result) in [
        (
            "ja-Latn-fonipa-hepburn-heploc",
            "und-hepburn-heploc",
            "und-alalc97",
            "ja-Latn-alalc97-fonipa",
        ),
        ("sgn-DD", "und-DD", "und-DE", "sgn-DE"),
        ("sgn-DE", "sgn-DE", "gsg", "gsg"),
    ] {
        let mut locale = locale.parse().unwrap();
        let rule_0 = rule_0.parse::<LanguageIdentifier>().unwrap();
        let rule_1 = rule_1.parse().unwrap();
        let result = result.parse::<Locale>().unwrap();
        uts35_replacement(
            &mut locale,
            !rule_0.language.is_empty(),
            rule_0.script.is_some(),
            rule_0.region.is_some(),
            Some(rule_0.variants.iter().map(Variant::as_str)),
            &rule_1,
        );
        assert_eq!(result, locale);
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	127✔
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	//! The collection of code for locale canonicalization.
6
7	use crate::provider::*;
8	use crate::LocaleTransformError;
9	use alloc::vec::Vec;
10	use core::cmp::Ordering;
11
12	use crate::LocaleExpander;
13	use crate::TransformResult;
14	use icu_locid::subtags::{Language, Region, Script};
15	use icu_locid::{
16	extensions::unicode::key,
17	subtags::{language, Variant, Variants},
18	LanguageIdentifier, Locale,
19	};
20	use icu_provider::prelude::*;
21	use tinystr::TinyAsciiStr;
22
23	/// Implements the algorithm defined in [UTS #35: Annex C, LocaleId Canonicalization].
24	///
25	/// # Examples
26	///
27	/// ```
28	/// use icu_locid::Locale;
29	/// use icu_locid_transform::{LocaleCanonicalizer, TransformResult};
30	///
31	/// let lc = LocaleCanonicalizer::new();
32	///
33	/// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap();
34	/// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified);
35	/// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap());
36	/// ```
37	///
38	/// [UTS #35: Annex C, LocaleId Canonicalization]: http://unicode.org/reports/tr35/#LocaleId_Canonicalization
39	#[derive(Debug)]	×
40	pub struct LocaleCanonicalizer {
41	/// Data to support canonicalization.
42	aliases: DataPayload<AliasesV1Marker>,	×
43	/// Likely subtags implementation for delegation.
44	expander: LocaleExpander,	×
45	}
46
47	#[inline]
48	fn uts35_rule_matches<'a, I>(	793✔
49	source: &Locale,
50	language: Language,
51	script: Option<Script>,
52	region: Option<Region>,
53	raw_variants: I,
54	) -> bool
55	where
56	I: Iterator<Item = &'a str>,
57	{
58	(language.is_empty() \|\| language == source.id.language)	1,431✔
59	&& (script.is_none() \|\| script == source.id.script)	301✔
60	&& (region.is_none() \|\| region == source.id.region)	301✔
61	&& {
62	// Checks if variants are a subset of source variants.
63	// As both iterators are sorted, this can be done linearly.
64	let mut source_variants = source.id.variants.iter();	301✔
65	'outer: for it in raw_variants {	566✔
66	for cand in source_variants.by_ref() {	580✔
67	match cand.strict_cmp(it.as_bytes()) {	372✔
68	Ordering::Equal => {
69	continue 'outer;
70	}
71	Ordering::Less => {}
72	_ => {
73	return false;	104✔
74	}
75	}
76	}
77	return false;	161✔
78	}
79	true
80	}
81	}	699✔
82
83	fn uts35_replacement<'a, I>(	51✔
84	source: &mut Locale,
85	ruletype_has_language: bool,
86	ruletype_has_script: bool,
87	ruletype_has_region: bool,
88	ruletype_variants: Option<I>,
89	replacement: &LanguageIdentifier,
90	) where
91	I: Iterator<Item = &'a str>,
92	{
93	if ruletype_has_language \|\| (source.id.language.is_empty() && !replacement.language.is_empty())	51✔
94	{
95	source.id.language = replacement.language;	28✔
96	}
97	if ruletype_has_script \|\| (source.id.script.is_none() && replacement.script.is_some()) {	51✔
98	source.id.script = replacement.script;	1✔
99	}
100	if ruletype_has_region \|\| (source.id.region.is_none() && replacement.region.is_some()) {	51✔
101	source.id.region = replacement.region;	8✔
102	}
103	if let Some(skips) = ruletype_variants {	87✔
104	// The rule matches if the ruletype variants are a subset of the source variants.
105	// This means ja-Latn-fonipa-hepburn-heploc matches against the rule for
106	// hepburn-heploc and is canonicalized to ja-Latn-alalc97-fonipa
107
108	// We're merging three sorted deduped iterators into a new sequence:
109	// sources - skips + replacements
110
111	let mut sources = source.id.variants.iter().copied().peekable();	36✔
112	let mut replacements = replacement.variants.iter().copied().peekable();	36✔
113	let mut skips = skips.peekable();	36✔
114
115	let mut variants: Vec<Variant> = Vec::new();	36✔
116
117	loop {	36✔
118	match (sources.peek(), skips.peek(), replacements.peek()) {	117✔
119	(Some(&source), Some(skip), _)	59✔
120	if source.strict_cmp(skip.as_bytes()) == Ordering::Greater =>	59✔
121	{
122	skips.next();	×
123	}
124	(Some(&source), Some(skip), _)	105✔
125	if source.strict_cmp(skip.as_bytes()) == Ordering::Equal =>	59✔
126	{
127	skips.next();	46✔
128	sources.next();	46✔
129	}
130	(Some(&source), _, Some(&replacement))	13✔
131	if replacement.cmp(&source) == Ordering::Less =>	8✔
132	{
133	variants.push(replacement);	5✔
134	replacements.next();	5✔
135	}
136	(Some(&source), _, Some(&replacement))	6✔
137	if replacement.cmp(&source) == Ordering::Equal =>	3✔
138	{
139	variants.push(source);	3✔
140	sources.next();	3✔
141	replacements.next();	3✔
142	}
143	(Some(&source), _, _) => {	10✔
144	variants.push(source);	10✔
145	sources.next();	10✔
146	}
147	(None, _, Some(&replacement)) => {	4✔
148	variants.push(replacement);	4✔
149	replacements.next();	4✔
150	}
151	(None, _, None) => {
152	break;
153	}
154	}
155	}
156	source.id.variants = Variants::from_vec_unchecked(variants);	36✔
157	}	36✔
158	}	51✔
159
160	#[inline]
161	fn uts35_check_language_rules(	98✔
162	locale: &mut Locale,
163	alias_data: &DataPayload<AliasesV1Marker>,
164	) -> TransformResult {
165	if !locale.id.language.is_empty() {	98✔
166	let lang: TinyAsciiStr<3> = locale.id.language.into();	98✔
167	let replacement = if lang.len() == 2 {	98✔
168	alias_data	146✔
169	.get()
170	.language_len2
171	.get(&lang.resize().to_unvalidated())	73✔
172	} else {
173	alias_data.get().language_len3.get(&lang.to_unvalidated())	25✔
174	};
175
176	if let Some(replacement) = replacement {	98✔
177	if let Ok(langid) = replacement.parse() {	13✔
178	uts35_replacement::<core::iter::Empty<&str>>(	13✔
179	locale, true, false, false, None, &langid,	13✔
180	);
181	return TransformResult::Modified;	13✔
182	}	13✔
183	}	×
184	}
185
186	TransformResult::Unmodified	85✔
187	}	98✔
188
189	fn is_iter_sorted<I, T>(mut iter: I) -> bool	695✔
190	where
191	I: Iterator<Item = T>,
192	T: PartialOrd,
193	{
194	if let Some(mut last) = iter.next() {	695✔
195	for curr in iter {	742✔
196	if last > curr {	47✔
197	return false;	×
198	}
199	last = curr;	47✔
200	}	47✔
201	}	695✔
202	true	695✔
203	}	695✔
204
205	#[cfg(feature = "compiled_data")]
206	impl Default for LocaleCanonicalizer {
207	fn default() -> Self {	×
208	Self::new()	×
209	}	×
210	}
211
212	impl LocaleCanonicalizer {
213	/// A constructor which creates a [`LocaleCanonicalizer`] from compiled data.
214	///
215	/// ✨ Enabled with the `compiled_data` Cargo feature.
216	///
217	/// [📚 Help choosing a constructor](icu_provider::constructors)
218	#[cfg(feature = "compiled_data")]
219	pub const fn new() -> Self {	4✔
220	Self::new_with_expander(LocaleExpander::new_extended())	4✔
221	}	4✔
222
223	// Note: This is a custom impl because the bounds on LocaleExpander::try_new_unstable changed
224	#[doc = icu_provider::gen_any_buffer_unstable_docs!(ANY, Self::new)]
225	pub fn try_new_with_any_provider(	×
226	provider: &(impl AnyProvider + ?Sized),
227	) -> Result<LocaleCanonicalizer, LocaleTransformError> {
228	let expander = LocaleExpander::try_new_with_any_provider(provider)?;	×
229	Self::try_new_with_expander_unstable(&provider.as_downcasting(), expander)	×
230	}	×
231
232	// Note: This is a custom impl because the bounds on LocaleExpander::try_new_unstable changed
233	#[doc = icu_provider::gen_any_buffer_unstable_docs!(BUFFER, Self::new)]
234	#[cfg(feature = "serde")]
235	pub fn try_new_with_buffer_provider(	×
236	provider: &(impl BufferProvider + ?Sized),
237	) -> Result<LocaleCanonicalizer, LocaleTransformError> {
238	let expander = LocaleExpander::try_new_with_buffer_provider(provider)?;	×
239	Self::try_new_with_expander_unstable(&provider.as_deserializing(), expander)	×
240	}	×
241
242	#[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new)]
243	pub fn try_new_unstable<P>(provider: &P) -> Result<LocaleCanonicalizer, LocaleTransformError>
244	where
245	P: DataProvider<AliasesV1Marker>
246	+ DataProvider<LikelySubtagsForLanguageV1Marker>
247	+ DataProvider<LikelySubtagsForScriptRegionV1Marker>
248	+ ?Sized,
249	{
250	let expander = LocaleExpander::try_new_unstable(provider)?;
251	Self::try_new_with_expander_unstable(provider, expander)
252	}
253
254	/// Creates a [`LocaleCanonicalizer`] with a custom [`LocaleExpander`] and compiled data.
255	///
256	/// ✨ Enabled with the `compiled_data` Cargo feature.
257	///
258	/// [📚 Help choosing a constructor](icu_provider::constructors)
259	#[cfg(feature = "compiled_data")]
260	pub const fn new_with_expander(expander: LocaleExpander) -> Self {	4✔
261	Self {	4✔
262	aliases: DataPayload::from_static_ref(	4✔
263	crate::provider::Baked::SINGLETON_LOCID_TRANSFORM_ALIASES_V1,
264	),
265	expander,	4✔
266	}
267	}	4✔
268
269	#[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_with_expander)]
270	pub fn try_new_with_expander_unstable<P>(	×
271	provider: &P,
272	expander: LocaleExpander,
273	) -> Result<LocaleCanonicalizer, LocaleTransformError>
274	where
275	P: DataProvider<AliasesV1Marker> + ?Sized,
276	{
277	let aliases: DataPayload<AliasesV1Marker> =
278	provider.load(Default::default())?.take_payload()?;	×
279
280	Ok(LocaleCanonicalizer { aliases, expander })	×
281	}	×
282
283	icu_provider::gen_any_buffer_data_constructors!(
284	locale: skip,
285	options: LocaleExpander,
286	error: LocaleTransformError,
287	#[cfg(skip)]
288	functions: [
289	new_with_expander,
290	try_new_with_expander_with_any_provider,
291	try_new_with_expander_with_buffer_provider,
292	try_new_with_expander_unstable,
293	Self,
294	]
295	);
296
297	/// The canonicalize method potentially updates a passed in locale in place
298	/// depending up the results of running the canonicalization algorithm
299	/// from <http://unicode.org/reports/tr35/#LocaleId_Canonicalization>.
300	///
301	/// Some BCP47 canonicalization data is not part of the CLDR json package. Because
302	/// of this, some canonicalizations are not performed, e.g. the canonicalization of
303	/// `und-u-ca-islamicc` to `und-u-ca-islamic-civil`. This will be fixed in a future
304	/// release once the missing data has been added to the CLDR json data. See:
305	/// <https://github.com/unicode-org/icu4x/issues/746>
306	///
307	/// # Examples
308	///
309	/// ```
310	/// use icu_locid::Locale;
311	/// use icu_locid_transform::{LocaleCanonicalizer, TransformResult};
312	///
313	/// let lc = LocaleCanonicalizer::new();
314	///
315	/// let mut locale: Locale = "ja-Latn-fonipa-hepburn-heploc".parse().unwrap();
316	/// assert_eq!(lc.canonicalize(&mut locale), TransformResult::Modified);
317	/// assert_eq!(locale, "ja-Latn-alalc97-fonipa".parse().unwrap());
318	/// ```
319	pub fn canonicalize(&self, locale: &mut Locale) -> TransformResult {	91✔
320	let mut result = TransformResult::Unmodified;	91✔
321
322	// This loops until we get a 'fixed point', where applying the rules do not
323	// result in any more changes.
324	'outer: loop {	91✔
325	// These are linear searches due to the ordering imposed by the canonicalization
326	// rules, where rules with more variants should be considered first. With the
327	// current data in CLDR, we will only do this for locales which have variants,
328	// or new rules which we haven't special-cased yet (of which there are fewer
329	// than 20).
330	if !locale.id.variants.is_empty() {	163✔
331	// These language/variant comibnations have around 20 rules
332	for StrStrPair(raw_lang_variants, raw_to) in self	751✔
333	.aliases
334	.get()
335	.language_variants
336	.iter()
337	.map(zerofrom::ZeroFrom::zero_from)
338	{
339	let (raw_lang, raw_variants) = {	695✔
340	let mut subtags = raw_lang_variants.split('-');	695✔
341	(	695✔
342	// str::split can't return empty iterators
343	unsafe { subtags.next().unwrap_unchecked() },	695✔
344	subtags,	695✔
345	)
346	};
347	if is_iter_sorted(raw_variants.clone()) {	695✔
348	if let Ok(lang) = raw_lang.parse::<Language>() {	695✔
349	if uts35_rule_matches(locale, lang, None, None, raw_variants.clone()) {	695✔
350	if let Ok(to) = raw_to.parse() {	33✔
351	uts35_replacement(	33✔
352	locale,
353	!lang.is_empty(),	33✔
354	false,
355	false,
356	Some(raw_variants),	33✔
357	&to,
358	);
359	result = TransformResult::Modified;	33✔
360	continue 'outer;
361	}	33✔
362	}	×
363	}
364	}
365	}	695✔
366	} else {
367	// These are absolute fallbacks, and currently empty.
368	for StrStrPair(raw_from, raw_to) in self	107✔
369	.aliases
370	.get()
371	.language
372	.iter()
373	.map(zerofrom::ZeroFrom::zero_from)
374	{
375	if let Ok(from) = raw_from.parse::<LanguageIdentifier>() {	×
376	if uts35_rule_matches(	×
377	locale,
378	from.language,	×
379	from.script,	×
380	from.region,	×
381	from.variants.iter().map(Variant::as_str),	×
382	) {
383	if let Ok(to) = raw_to.parse() {	×
384	uts35_replacement(	×
385	locale,
386	!from.language.is_empty(),	×
387	from.script.is_some(),	×
388	from.region.is_some(),	×
389	Some(from.variants.iter().map(Variant::as_str)),	×
390	&to,
391	);
392	result = TransformResult::Modified;	×
393	continue 'outer;
394	}	×
395	}	×
396	}	×
397	}	×
398	}
399
400	if !locale.id.language.is_empty() {	130✔
401	// If the region is specified, check sgn-region rules first
402	if let Some(region) = locale.id.region {	97✔
403	if locale.id.language == language!("sgn") {	37✔
404	if let Some(&sgn_lang) = self	6✔
405	.aliases
406	.get()
407	.sgn_region
408	.get(&region.into_tinystr().to_unvalidated())	3✔
409	{
410	uts35_replacement::<core::iter::Empty<&str>>(	2✔
411	locale,
412	true,
413	false,
414	true,
415	None,	2✔
416	&sgn_lang.into(),	2✔
417	);	2✔
418	result = TransformResult::Modified;	2✔
419	continue;
420	}
421	}
422	}
423
424	if uts35_check_language_rules(locale, &self.aliases) == TransformResult::Modified {	95✔
425	result = TransformResult::Modified;	12✔
426	continue;
427	}
428	}
429
430	if let Some(script) = locale.id.script {	116✔
431	if let Some(&replacement) = self	74✔
432	.aliases
433	.get()
434	.script
435	.get(&script.into_tinystr().to_unvalidated())	37✔
436	{
437	locale.id.script = Some(replacement);	1✔
438	result = TransformResult::Modified;	1✔
439	continue;
440	}
441	}
442
443	if let Some(region) = locale.id.region {	115✔
444	let replacement = if region.is_alphabetic() {	54✔
445	self.aliases	92✔
446	.get()
447	.region_alpha
448	.get(&region.into_tinystr().resize().to_unvalidated())	46✔
449	} else {
450	self.aliases	16✔
451	.get()
452	.region_num
453	.get(&region.into_tinystr().to_unvalidated())	8✔
454	};
455	if let Some(&replacement) = replacement {	54✔
456	locale.id.region = Some(replacement);	3✔
457	result = TransformResult::Modified;	3✔
458	continue;
459	}
460
461	if let Some(regions) = self	102✔
462	.aliases
463	.get()
464	.complex_region
465	.get(&region.into_tinystr().to_unvalidated())	51✔
466	{
467	// Skip if regions are empty
468	if let Some(default_region) = regions.get(0) {	17✔
469	let mut maximized = LanguageIdentifier {	17✔
470	language: locale.id.language,	17✔
471	script: locale.id.script,	17✔
472	region: None,	17✔
473	variants: Variants::default(),	17✔
474	};
475
476	locale.id.region = Some(	17✔
477	match (self.expander.maximize(&mut maximized), maximized.region) {	17✔
478	(TransformResult::Modified, Some(candidate))	27✔
479	if regions.iter().any(\|x\| x == candidate) =>	124✔
480	{
481	candidate	10✔
482	}	10✔
483	_ => default_region,	7✔
484	},
485	);
486	result = TransformResult::Modified;	17✔
487	continue;
488	}	17✔
489	}
490	}
491
492	if !locale.id.variants.is_empty() {	114✔
493	let mut modified = Vec::new();	23✔
494	let mut unmodified = Vec::new();	23✔
495	for &variant in locale.id.variants.iter() {	52✔
496	if let Some(&updated) = self	58✔
497	.aliases
498	.get()
499	.variant
500	.get(&variant.into_tinystr().to_unvalidated())	29✔
501	{
502	modified.push(updated);	5✔
503	} else {
504	unmodified.push(variant);	24✔
505	}
506	}
507
508	if !modified.is_empty() {	23✔
509	modified.extend(unmodified);	4✔
510	modified.sort();	4✔
511	modified.dedup();	4✔
512	locale.id.variants = Variants::from_vec_unchecked(modified);	4✔
513	result = TransformResult::Modified;	4✔
514	continue;
515	}
516	}	23✔
517
518	// Nothing matched in this iteration, we're done.
519	break;
520	}
521
522	// Handle Locale extensions in their own loops, because these rules do not interact
523	// with each other.
524	if let Some(lang) = &locale.extensions.transform.lang {	93✔
525	let mut tlang: Locale = lang.clone().into();	2✔
526	let mut matched = false;	2✔
527	loop {	2✔
528	if uts35_check_language_rules(&mut tlang, &self.aliases)	3✔
529	== TransformResult::Modified
530	{
531	result = TransformResult::Modified;	1✔
532	matched = true;	1✔
533	continue;
534	}
535
536	break;
537	}
538
539	if matched {	3✔
540	locale.extensions.transform.lang = Some(tlang.id);	1✔
541	}
542	}	2✔
543
544	// The `rg` region override and `sd` regional subdivision keys may contain
545	// language codes that require canonicalization.
546	for key in &[key!("rg"), key!("sd")] {	273✔
547	if let Some(value) = locale.extensions.unicode.keywords.get_mut(key) {	182✔
548	if let &[only_value] = value.as_tinystr_slice() {	16✔
549	if let Some(modified_value) = self	48✔
550	.aliases
551	.get()
552	.subdivision
553	.get(&only_value.resize().to_unvalidated())	16✔
554	{
555	if let Ok(modified_value) = modified_value.parse() {	16✔
556	*value = modified_value;	16✔
557	result = TransformResult::Modified;	16✔
558	}
559	}	16✔
560	}
561	}
562	}
563
564	result
565	}	91✔
566	}
567
568	#[test]
569	fn test_uts35_rule_matches() {	2✔
570	for (source, rule, result) in [	1✔
571	("ja", "und", true),	1✔
572	("und-heploc-hepburn", "und-hepburn", true),	1✔
573	("ja-heploc-hepburn", "und-hepburn", true),	1✔
574	("ja-hepburn", "und-hepburn-heploc", false),	1✔
575	] {
576	let source = source.parse().unwrap();	4✔
577	let rule = rule.parse::<LanguageIdentifier>().unwrap();	4✔
578	assert_eq!(	4✔
579	uts35_rule_matches(	4✔
580	&source,
581	rule.language,	4✔
582	rule.script,	4✔
583	rule.region,	4✔
584	rule.variants.iter().map(Variant::as_str),	4✔
585	),
586	result,
587	"{source}"
588	);
589	}	4✔
590	}	2✔
591
592	#[test]
593	fn test_uts35_replacement() {	2✔
594	for (locale, rule_0, rule_1, result) in [	1✔
595	(	1✔
596	"ja-Latn-fonipa-hepburn-heploc",
597	"und-hepburn-heploc",
598	"und-alalc97",
599	"ja-Latn-alalc97-fonipa",
600	),
601	("sgn-DD", "und-DD", "und-DE", "sgn-DE"),	1✔
602	("sgn-DE", "sgn-DE", "gsg", "gsg"),	1✔
603	] {
604	let mut locale = locale.parse().unwrap();	3✔
605	let rule_0 = rule_0.parse::<LanguageIdentifier>().unwrap();	3✔
606	let rule_1 = rule_1.parse().unwrap();	3✔
607	let result = result.parse::<Locale>().unwrap();	3✔
608	uts35_replacement(	3✔
609	&mut locale,
610	!rule_0.language.is_empty(),	3✔
611	rule_0.script.is_some(),	3✔
612	rule_0.region.is_some(),	3✔
613	Some(rule_0.variants.iter().map(Variant::as_str)),	3✔
614	&rule_1,
615	);
616	assert_eq!(result, locale);	3✔
617	}	3✔
618	}	2✔

zbraniecki / icu4x / 6815798908

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous