9281145371

Committed 29 May 2024 06:11AM UTC coverage: 76.254% (+0.1%) from 76.113%

Build # 9281145371

Build Type

push

github

Committed by

web-flow

Commit Message

Move generic Subtag to subtags (#4932)

First patch from the #1833 patchset.

It moves the `Subtag` to `locid::subtags::Subtag` - this is a generic
subtag (2..=8, separate from private::Subtag which is 1..=8) which will
be used to iterate over multi-part subtags like
`extensions::unicode::Value` to retrieve specific items without relying
on the underlying `TinyStr` storage.

It's a breaking change because we are removing
`extensions::other::Subtag`. I assume this is ok as we're working on 2.0
so I don't need to add alias and deprecate it, but please confirm.

Run Details

16 of 16 new or added lines in 1 file covered. (100.0%)

1507 existing lines in 83 files now uncovered.

53859 of 70631 relevant lines covered (76.25%)

578959.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.1

/components/list/src/patterns.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use crate::lazy_automaton::LazyAutomaton;
use crate::provider::*;
use crate::ListLength;
#[cfg(feature = "datagen")]
use alloc::borrow::Cow;
#[cfg(feature = "datagen")]
use icu_provider::DataError;
use writeable::{LengthHint, Writeable};

impl<'data> ListFormatterPatternsV1<'data> {
    /// Creates a new [`ListFormatterPatternsV1`] from the given patterns. Fails if any pattern is invalid.
    #[cfg(feature = "datagen")]
    pub fn try_new(
        [start, middle, end, pair, short_start, short_middle, short_end, short_pair, narrow_start, narrow_middle, narrow_end, narrow_pair]: [&str; 12],
    ) -> Result<Self, DataError> {
        Ok(Self([
            ListJoinerPattern::from_str(start, true, false)?.into(),
            ListJoinerPattern::from_str(middle, false, false)?.into(),
            ListJoinerPattern::from_str(end, false, true)?.into(),
            ListJoinerPattern::from_str(pair, true, true)?.into(),
            ListJoinerPattern::from_str(short_start, true, false)?.into(),
            ListJoinerPattern::from_str(short_middle, false, false)?.into(),
            ListJoinerPattern::from_str(short_end, false, true)?.into(),
            ListJoinerPattern::from_str(short_pair, true, true)?.into(),
            ListJoinerPattern::from_str(narrow_start, true, false)?.into(),
            ListJoinerPattern::from_str(narrow_middle, false, false)?.into(),
            ListJoinerPattern::from_str(narrow_end, false, true)?.into(),
            ListJoinerPattern::from_str(narrow_pair, true, true)?.into(),
        ]))
    }

    /// Adds a special case to all `pattern`s that will evaluate to
    /// `alternative_pattern` when `regex` matches the following element.
    /// The regex is interpreted case-insensitive and anchored to the beginning, but
    /// to improve efficiency does not search for full matches. If a full match is
    /// required, use `$`.
    #[cfg(feature = "datagen")]
    pub fn make_conditional(
        &mut self,
        pattern: &str,
        regex: &SerdeDFA<'static>,
        alternative_pattern: &str,
    ) -> Result<(), DataError> {
        let old = ListJoinerPattern::from_str(pattern, true, true)?;
        for i in 0..12 {
            #[allow(clippy::indexing_slicing)] // self.0 is &[_; 12]
            if self.0[i].default == old {
                self.0[i].special_case = Some(SpecialCasePattern {
                    condition: regex.clone(),
                    pattern: ListJoinerPattern::from_str(
                        alternative_pattern,
                        i % 4 == 0 || i % 4 == 3, // allow_prefix = start or pair
                        i % 4 == 2 || i % 4 == 3, // allow_suffix = end or pair
                    )?,
                });
            }
        }
        Ok(())
    }

    /// The range of the number of bytes required by the list literals to join a
    /// list of length `len`. If none of the patterns are conditional, this is exact.
    pub(crate) fn size_hint(&self, style: ListLength, len: usize) -> LengthHint {
        match len {
            0 | 1 => LengthHint::exact(0),
            2 => self.pair(style).size_hint(),
            n => {
                self.start(style).size_hint()
                    + self.middle(style).size_hint() * (n - 3)
                    + self.end(style).size_hint()
            }
        }
    }
}

type PatternParts<'a> = (&'a str, &'a str, &'a str);

impl<'a> ConditionalListJoinerPattern<'a> {
    pub(crate) fn parts<'b, W: Writeable + ?Sized>(
        &'a self,
        following_value: &'b W,
    ) -> PatternParts<'a> {
        match &self.special_case {
            Some(SpecialCasePattern { condition, pattern })
                if condition.deref().matches_earliest_fwd_lazy(following_value) =>
            {
                pattern.borrow_tuple()
            }
            _ => self.default.borrow_tuple(),
        }
    }

    /// The expected length of this pattern
    fn size_hint(&'a self) -> LengthHint {
        let mut hint = self.default.size_hint();
        if let Some(special_case) = &self.special_case {
            hint |= special_case.pattern.size_hint()
        }
        hint
    }
}

impl<'data> ListJoinerPattern<'data> {
    #[cfg(feature = "datagen")]
    fn from_str(pattern: &str, allow_prefix: bool, allow_suffix: bool) -> Result<Self, DataError> {
        match (pattern.find("{0}"), pattern.find("{1}")) {
            (Some(index_0), Some(index_1))
                if index_0 < index_1
                    && (allow_prefix || index_0 == 0)
                    && (allow_suffix || index_1 == pattern.len() - 3) =>
            {
                if (index_0 > 0 && !cfg!(test)) || index_1 - 3 >= 256 {
                    return Err(DataError::custom(
                        "Found valid pattern that cannot be stored in ListFormatterPatternsV1",
                    )
                    .with_debug_context(pattern));
                }
                #[allow(clippy::indexing_slicing)] // find
                Ok(ListJoinerPattern {
                    string: Cow::Owned(alloc::format!(
                        "{}{}{}",
                        &pattern[0..index_0],
                        &pattern[index_0 + 3..index_1],
                        &pattern[index_1 + 3..]
                    )),
                    index_0: index_0 as u8,
                    index_1: (index_1 - 3) as u8,
                })
            }
            _ => Err(DataError::custom("Invalid list pattern").with_debug_context(pattern)),
        }
    }

    fn borrow_tuple(&'data self) -> PatternParts<'data> {
        #![allow(clippy::indexing_slicing)] // by invariant
        let index_0 = self.index_0 as usize;
        let index_1 = self.index_1 as usize;
        (
            &self.string[0..index_0],
            &self.string[index_0..index_1],
            &self.string[index_1..],
        )
    }

    fn size_hint(&self) -> LengthHint {
        LengthHint::exact(self.string.len())
    }
}

#[cfg(feature = "datagen")]
impl<'data> From<ListJoinerPattern<'data>> for ConditionalListJoinerPattern<'data> {
    fn from(default: ListJoinerPattern<'data>) -> Self {
        Self {
            default,
            special_case: None,
        }
    }
}

#[cfg(all(test, feature = "datagen"))]
pub mod test {
    use super::*;

    pub fn test_patterns() -> ListFormatterPatternsV1<'static> {
        let mut patterns = ListFormatterPatternsV1::try_new([
            // Wide: general
            "@{0}:{1}",
            "{0},{1}",
            "{0}.{1}!",
            "${0};{1}+",
            // Short: different pattern lengths
            "{0}1{1}",
            "{0}12{1}",
            "{0}12{1}34",
            "{0}123{1}456",
            // Narrow: conditionals
            "{0}: {1}",
            "{0}, {1}",
            "{0}. {1}",
            "{0}. {1}",
        ])
        .unwrap();
        patterns
            .make_conditional(
                "{0}. {1}",
                &SerdeDFA::new(Cow::Borrowed("A")).unwrap(),
                "{0} :o {1}",
            )
            .unwrap();
        patterns
    }

    #[test]
    fn rejects_bad_patterns() {
        assert!(ListJoinerPattern::from_str("{0} and", true, true).is_err());
        assert!(ListJoinerPattern::from_str("and {1}", true, true).is_err());
        assert!(ListJoinerPattern::from_str("{1} and {0}", true, true).is_err());
        assert!(ListJoinerPattern::from_str("{1{0}}", true, true).is_err());
        assert!(ListJoinerPattern::from_str("{0\u{202e}} and {1}", true, true).is_err());
        assert!(ListJoinerPattern::from_str("{{0}} {{1}}", true, true).is_ok());

        assert!(ListJoinerPattern::from_str("{0} and {1} ", true, true).is_ok());
        assert!(ListJoinerPattern::from_str("{0} and {1} ", true, false).is_err());
        assert!(ListJoinerPattern::from_str(" {0} and {1}", true, true).is_ok());
        assert!(ListJoinerPattern::from_str(" {0} and {1}", false, true).is_err());
    }

    #[test]
    fn produces_correct_parts() {
        assert_eq!(
            test_patterns().pair(ListLength::Wide).parts(""),
            ("$", ";", "+")
        );
    }

    #[test]
    fn produces_correct_parts_conditionally() {
        assert_eq!(
            test_patterns().end(ListLength::Narrow).parts("A"),
            ("", " :o ", "")
        );
        assert_eq!(
            test_patterns().end(ListLength::Narrow).parts("a"),
            ("", " :o ", "")
        );
        assert_eq!(
            test_patterns().end(ListLength::Narrow).parts("ab"),
            ("", " :o ", "")
        );
        assert_eq!(
            test_patterns().end(ListLength::Narrow).parts("B"),
            ("", ". ", "")
        );
        assert_eq!(
            test_patterns().end(ListLength::Narrow).parts("BA"),
            ("", ". ", "")
        );
    }

    #[test]
    fn size_hint_works() {
        let pattern = test_patterns();

        assert_eq!(
            pattern.size_hint(ListLength::Short, 0),
            LengthHint::exact(0)
        );
        assert_eq!(
            pattern.size_hint(ListLength::Short, 1),
            LengthHint::exact(0)
        );

        // pair pattern "{0}123{1}456"
        assert_eq!(
            pattern.size_hint(ListLength::Short, 2),
            LengthHint::exact(6)
        );

        // patterns "{0}1{1}", "{0}12{1}" (x197), and "{0}12{1}34"
        assert_eq!(
            pattern.size_hint(ListLength::Short, 200),
            LengthHint::exact(1 + 2 * 197 + 4)
        );

        // patterns "{0}: {1}", "{0}, {1}" (x197), and "{0} :o {1}" or "{0}. {1}"
        assert_eq!(
            pattern.size_hint(ListLength::Narrow, 200),
            LengthHint::exact(2 + 197 * 2) + LengthHint::between(2, 4)
        );
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	×
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	use crate::lazy_automaton::LazyAutomaton;
6	use crate::provider::*;
7	use crate::ListLength;
8	#[cfg(feature = "datagen")]
9	use alloc::borrow::Cow;
10	#[cfg(feature = "datagen")]
11	use icu_provider::DataError;
12	use writeable::{LengthHint, Writeable};
13
14	impl<'data> ListFormatterPatternsV1<'data> {
15	/// Creates a new [`ListFormatterPatternsV1`] from the given patterns. Fails if any pattern is invalid.
16	#[cfg(feature = "datagen")]
17	pub fn try_new(	65✔
18	[start, middle, end, pair, short_start, short_middle, short_end, short_pair, narrow_start, narrow_middle, narrow_end, narrow_pair]: [&str; 12],	65✔
19	) -> Result<Self, DataError> {
20	Ok(Self([	65✔
21	ListJoinerPattern::from_str(start, true, false)?.into(),	65✔
22	ListJoinerPattern::from_str(middle, false, false)?.into(),	65✔
23	ListJoinerPattern::from_str(end, false, true)?.into(),	65✔
24	ListJoinerPattern::from_str(pair, true, true)?.into(),	65✔
25	ListJoinerPattern::from_str(short_start, true, false)?.into(),	65✔
26	ListJoinerPattern::from_str(short_middle, false, false)?.into(),	65✔
27	ListJoinerPattern::from_str(short_end, false, true)?.into(),	65✔
28	ListJoinerPattern::from_str(short_pair, true, true)?.into(),	65✔
29	ListJoinerPattern::from_str(narrow_start, true, false)?.into(),	65✔
30	ListJoinerPattern::from_str(narrow_middle, false, false)?.into(),	65✔
31	ListJoinerPattern::from_str(narrow_end, false, true)?.into(),	65✔
32	ListJoinerPattern::from_str(narrow_pair, true, true)?.into(),	65✔
UNCOV 33	]))	×
34	}	65✔
35
36	/// Adds a special case to all `pattern`s that will evaluate to
37	/// `alternative_pattern` when `regex` matches the following element.
38	/// The regex is interpreted case-insensitive and anchored to the beginning, but
39	/// to improve efficiency does not search for full matches. If a full match is
40	/// required, use `$`.
41	#[cfg(feature = "datagen")]
42	pub fn make_conditional(	27✔
43	&mut self,
44	pattern: &str,
45	regex: &SerdeDFA<'static>,
46	alternative_pattern: &str,
47	) -> Result<(), DataError> {
48	let old = ListJoinerPattern::from_str(pattern, true, true)?;	27✔
49	for i in 0..12 {	223✔
50	#[allow(clippy::indexing_slicing)] // self.0 is &[_; 12]
51	if self.0[i].default == old {	251✔
52	self.0[i].special_case = Some(SpecialCasePattern {	52✔
53	condition: regex.clone(),	54✔
54	pattern: ListJoinerPattern::from_str(	52✔
55	alternative_pattern,
56	i % 4 == 0 \|\| i % 4 == 3, // allow_prefix = start or pair	52✔
57	i % 4 == 2 \|\| i % 4 == 3, // allow_suffix = end or pair	52✔
UNCOV 58	)?,	×
59	});	52✔
60	}
61	}
62	Ok(())	17✔
63	}	17✔
64
65	/// The range of the number of bytes required by the list literals to join a
66	/// list of length `len`. If none of the patterns are conditional, this is exact.
67	pub(crate) fn size_hint(&self, style: ListLength, len: usize) -> LengthHint {	77✔
68	match len {	77✔
69	0 \| 1 => LengthHint::exact(0),	6✔
70	2 => self.pair(style).size_hint(),	57✔
71	n => {	14✔
72	self.start(style).size_hint()	42✔
73	+ self.middle(style).size_hint() * (n - 3)	14✔
74	+ self.end(style).size_hint()	14✔
75	}
76	}
77	}	77✔
78	}
79
80	type PatternParts<'a> = (&'a str, &'a str, &'a str);
81
82	impl<'a> ConditionalListJoinerPattern<'a> {
83	pub(crate) fn parts<'b, W: Writeable + ?Sized>(	217✔
84	&'a self,
85	following_value: &'b W,
86	) -> PatternParts<'a> {
87	match &self.special_case {	217✔
88	Some(SpecialCasePattern { condition, pattern })	138✔
89	if condition.deref().matches_earliest_fwd_lazy(following_value) =>	138✔
90	{
91	pattern.borrow_tuple()	100✔
92	}
93	_ => self.default.borrow_tuple(),	117✔
94	}
95	}	217✔
96
97	/// The expected length of this pattern
98	fn size_hint(&'a self) -> LengthHint {	99✔
99	let mut hint = self.default.size_hint();	99✔
100	if let Some(special_case) = &self.special_case {	99✔
101	hint \|= special_case.pattern.size_hint()	49✔
102	}
103	hint
104	}	99✔
105	}
106
107	impl<'data> ListJoinerPattern<'data> {
108	#[cfg(feature = "datagen")]
109	fn from_str(pattern: &str, allow_prefix: bool, allow_suffix: bool) -> Result<Self, DataError> {	858✔
110	match (pattern.find("{0}"), pattern.find("{1}")) {	858✔
111	(Some(index_0), Some(index_1))	1,706✔
112	if index_0 < index_1	855✔
113	&& (allow_prefix \|\| index_0 == 0)	854✔
114	&& (allow_suffix \|\| index_1 == pattern.len() - 3) =>	852✔
115	{
116	if (index_0 > 0 && !cfg!(test)) \|\| index_1 - 3 >= 256 {	851✔
UNCOV 117	return Err(DataError::custom(	×
118	"Found valid pattern that cannot be stored in ListFormatterPatternsV1",
119	)
120	.with_debug_context(pattern));
121	}
122	#[allow(clippy::indexing_slicing)] // find
123	Ok(ListJoinerPattern {	851✔
124	string: Cow::Owned(alloc::format!(	2,553✔
125	"{}{}{}",
126	&pattern[0..index_0],	851✔
127	&pattern[index_0 + 3..index_1],	851✔
128	&pattern[index_1 + 3..]	851✔
129	)),
130	index_0: index_0 as u8,	851✔
131	index_1: (index_1 - 3) as u8,	851✔
132	})
133	}	851✔
134	_ => Err(DataError::custom("Invalid list pattern").with_debug_context(pattern)),	7✔
135	}
136	}	858✔
137
138	fn borrow_tuple(&'data self) -> PatternParts<'data> {	271✔
139	#![allow(clippy::indexing_slicing)] // by invariant
140	let index_0 = self.index_0 as usize;	271✔
141	let index_1 = self.index_1 as usize;	271✔
142	(	271✔
143	&self.string[0..index_0],	271✔
144	&self.string[index_0..index_1],	271✔
145	&self.string[index_1..],	271✔
146	)
147	}	271✔
148
149	fn size_hint(&self) -> LengthHint {	148✔
150	LengthHint::exact(self.string.len())	148✔
151	}	148✔
152	}
153
154	#[cfg(feature = "datagen")]
155	impl<'data> From<ListJoinerPattern<'data>> for ConditionalListJoinerPattern<'data> {
156	fn from(default: ListJoinerPattern<'data>) -> Self {	780✔
157	Self {	780✔
158	default,	780✔
159	special_case: None,	780✔
160	}
161	}	780✔
162	}
163
164	#[cfg(all(test, feature = "datagen"))]
165	pub mod test {
166	use super::*;
167
168	pub fn test_patterns() -> ListFormatterPatternsV1<'static> {	11✔
169	let mut patterns = ListFormatterPatternsV1::try_new([	11✔
170	// Wide: general
171	"@{0}:{1}",
172	"{0},{1}",
173	"{0}.{1}!",
174	"${0};{1}+",
175	// Short: different pattern lengths
176	"{0}1{1}",
177	"{0}12{1}",
178	"{0}12{1}34",
179	"{0}123{1}456",
180	// Narrow: conditionals
181	"{0}: {1}",
182	"{0}, {1}",
183	"{0}. {1}",
184	"{0}. {1}",
185	])
186	.unwrap();
187	patterns	11✔
188	.make_conditional(
189	"{0}. {1}",
190	&SerdeDFA::new(Cow::Borrowed("A")).unwrap(),	11✔
191	"{0} :o {1}",
192	)
193	.unwrap();	11✔
194	patterns
195	}	11✔
196
197	#[test]
198	fn rejects_bad_patterns() {	2✔
199	assert!(ListJoinerPattern::from_str("{0} and", true, true).is_err());	1✔
200	assert!(ListJoinerPattern::from_str("and {1}", true, true).is_err());	1✔
201	assert!(ListJoinerPattern::from_str("{1} and {0}", true, true).is_err());	1✔
202	assert!(ListJoinerPattern::from_str("{1{0}}", true, true).is_err());	1✔
203	assert!(ListJoinerPattern::from_str("{0\u{202e}} and {1}", true, true).is_err());	1✔
204	assert!(ListJoinerPattern::from_str("{{0}} {{1}}", true, true).is_ok());	1✔
205
206	assert!(ListJoinerPattern::from_str("{0} and {1} ", true, true).is_ok());	1✔
207	assert!(ListJoinerPattern::from_str("{0} and {1} ", true, false).is_err());	1✔
208	assert!(ListJoinerPattern::from_str(" {0} and {1}", true, true).is_ok());	1✔
209	assert!(ListJoinerPattern::from_str(" {0} and {1}", false, true).is_err());	1✔
210	}	2✔
211
212	#[test]
213	fn produces_correct_parts() {	2✔
214	assert_eq!(	1✔
215	test_patterns().pair(ListLength::Wide).parts(""),	1✔
216	("$", ";", "+")
217	);
218	}	2✔
219
220	#[test]
221	fn produces_correct_parts_conditionally() {	2✔
222	assert_eq!(	1✔
223	test_patterns().end(ListLength::Narrow).parts("A"),	1✔
224	("", " :o ", "")
225	);
226	assert_eq!(	1✔
227	test_patterns().end(ListLength::Narrow).parts("a"),	1✔
228	("", " :o ", "")
229	);
230	assert_eq!(	1✔
231	test_patterns().end(ListLength::Narrow).parts("ab"),	1✔
232	("", " :o ", "")
233	);
234	assert_eq!(	1✔
235	test_patterns().end(ListLength::Narrow).parts("B"),	1✔
236	("", ". ", "")
237	);
238	assert_eq!(	1✔
239	test_patterns().end(ListLength::Narrow).parts("BA"),	1✔
240	("", ". ", "")
241	);
242	}	2✔
243
244	#[test]
245	fn size_hint_works() {	2✔
246	let pattern = test_patterns();	1✔
247
248	assert_eq!(	1✔
249	pattern.size_hint(ListLength::Short, 0),	1✔
250	LengthHint::exact(0)	1✔
251	);
252	assert_eq!(	1✔
253	pattern.size_hint(ListLength::Short, 1),	1✔
254	LengthHint::exact(0)	1✔
255	);
256
257	// pair pattern "{0}123{1}456"
258	assert_eq!(	1✔
259	pattern.size_hint(ListLength::Short, 2),	1✔
260	LengthHint::exact(6)	1✔
261	);
262
263	// patterns "{0}1{1}", "{0}12{1}" (x197), and "{0}12{1}34"
264	assert_eq!(	1✔
265	pattern.size_hint(ListLength::Short, 200),	1✔
266	LengthHint::exact(1 + 2 * 197 + 4)	1✔
267	);
268
269	// patterns "{0}: {1}", "{0}, {1}" (x197), and "{0} :o {1}" or "{0}. {1}"
270	assert_eq!(	1✔
271	pattern.size_hint(ListLength::Narrow, 200),	1✔
272	LengthHint::exact(2 + 197 * 2) + LengthHint::between(2, 4)	1✔
273	);
274	}	2✔
275	}

zbraniecki / icu4x / 9281145371

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous