9357137046

Committed 03 Jun 2024 08:51PM UTC coverage: 75.121% (-1.1%) from 76.254%

Build # 9357137046

Build Type

push

github

Committed by

web-flow

Commit Message

Switch locid Value to use Subtag (#4941)

This is part of #1833 switching Value API to use Subtag.

Run Details

61 of 71 new or added lines in 11 files covered. (85.92%)

3224 existing lines in 178 files now uncovered.

52958 of 70497 relevant lines covered (75.12%)

572757.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.37

/utils/deduplicating_array/src/lib.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! A serde serialization strategy that uses `PartialEq` to reduce serialized size.
//!
//! This create can be used with Serde derive like this:
//!
//! ```rust
//! # #[derive(Clone, PartialEq, serde::Deserialize, serde::Serialize)]
//! # struct Bar(String);
//!
//! #[derive(serde::Deserialize, serde::Serialize)]
//! pub struct Foo {
//!     #[serde(with = "deduplicating_array")]
//!     data: [Bar; 12],
//!     // ...
//! }
//! ```
//!
//! `Bar`s that are equal to a `Bar`s that appears earlier in the array will not be serialized
//! (instead, the index of the first occurence is serialized). Deserialization clones the first
//! `Bar` into all the indices where it occurs (hence `Bar` has to implement `Clone`).
//!
//! Human readable serialization represents skipped values as singleton arrays containing the
//! target index, e.g. the Rust array `["Foo", "Bar", "Foo"]` will serialize to JSON `["Foo", "Bar", [0]]`.
//!
//! This implies that singleton integer arrays cannot be used as array elements (they do work in Bincode,
//! but there's really not much point in using them).

// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
#![cfg_attr(not(test), no_std)]
#![cfg_attr(
    not(test),
    deny(
        clippy::indexing_slicing,
        clippy::unwrap_used,
        clippy::expect_used,
        clippy::panic,
        clippy::exhaustive_structs,
        clippy::exhaustive_enums,
        missing_debug_implementations,
    )
)]

extern crate alloc;

use alloc::fmt::{Error, Formatter};
use alloc::format;
use serde::de::{Deserialize, Deserializer, EnumAccess, Visitor};
use serde::ser::{Serialize, Serializer};

pub fn serialize<S, T, const N: usize>(array: &[T; N], serializer: S) -> Result<S::Ok, S::Error>
where
    S: Serializer,
    T: Serialize + PartialEq,
{
    use serde::ser::SerializeTuple;

    let human = serializer.is_human_readable();

    let mut seq = serializer.serialize_tuple(N)?;

    for i in 0..N {
        #[allow(clippy::indexing_slicing)] // i, j in 0..N
        match array.iter().take(i).position(|item| item == &array[i]) {
            None if human => seq.serialize_element(&HumanSer::Value(&array[i]))?,
            None => seq.serialize_element(&MachineSer::Value(&array[i]))?,
            Some(j) if human => seq.serialize_element(&HumanSer::<T>::Fallback([j]))?,
            Some(j) => seq.serialize_element(&MachineSer::<T>::Fallback(j))?,
        }
    }
    seq.end()
}

pub fn deserialize<'de, D, T, const N: usize>(deserializer: D) -> Result<[T; N], D::Error>
where
    D: Deserializer<'de>,
    T: Deserialize<'de> + Clone,
    [HumanDe<T>; N]: Deserialize<'de>,
    [MachineDe<T>; N]: Deserialize<'de>,
{
    use core::mem::MaybeUninit;
    use serde::de::Error;

    let mut array: [MaybeUninit<T>; N] = unsafe { MaybeUninit::uninit().assume_init() };

    if deserializer.is_human_readable() {
        for (i, r) in <[HumanDe<T>; N]>::deserialize(deserializer)?
            .into_iter()
            .enumerate()
        {
            match r {
                HumanDe::Value(v) => {
                    #[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
                    array[i].write(v);
                }
                HumanDe::Fallback([j]) => unsafe {
                    // Fallbacks should always be to a previous value,
                    // which makes the assume_init_ref safe
                    if j >= i {
                        return Err(D::Error::custom(format!(
                            "Illegal forward fallback {i}->{j}",
                        )));
                    }
                    #[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
                    array[i].write(array[j].assume_init_ref().clone());
                },
            }
        }
    } else {
        for (i, r) in
            IntoIterator::into_iter(<[MachineDe<T>; N]>::deserialize(deserializer)?).enumerate()
        {
            match r {
                MachineDe::Value(v) => {
                    #[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
                    array[i].write(v);
                }
                MachineDe::Fallback(j) => unsafe {
                    // Fallbacks should always be to a previous value,
                    // which makes the assume_init_ref safe
                    if j >= i {
                        return Err(D::Error::custom(format!(
                            "Illegal forward fallback {i}->{j}",
                        )));
                    }
                    #[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
                    array[i].write(array[j].assume_init_ref().clone());
                },
            }
        }
    }

    // https://github.com/rust-lang/rust/issues/61956
    Ok(unsafe { core::ptr::read(array.as_ptr() as *const [T; N]) })
}

#[derive(serde::Serialize)]
#[serde(untagged)]
enum HumanSer<'a, T> {
    Value(&'a T),
    Fallback([usize; 1]),
}

enum MachineSer<'a, T> {
    Value(&'a T),
    Fallback(usize),
}

// These enums are public because they are exposed in the trait bounds of
// deserialize. Serde only implements Deserialize<'de> on arrays up to
// size 32, so [Dedupe<T>;N]: Deserialize<'de> cannot be inferred from
// Dedupe<T>: Deserialize<'de> in the general case. See
// https://github.com/serde-rs/serde/issues/1937.
//
// These are not considered part of the stable API.

#[derive(serde::Deserialize, Debug)]
#[serde(untagged)]
#[doc(hidden)]
#[allow(clippy::exhaustive_enums)] // internal type
pub enum HumanDe<T> {
    Value(T),
    Fallback([usize; 1]),
}

#[doc(hidden)]
#[derive(Debug)]
#[allow(clippy::exhaustive_enums)] // internal type
pub enum MachineDe<T> {
    Value(T),
    Fallback(usize),
}

impl<'a, T> Serialize for MachineSer<'a, T>
where
    T: Serialize,
{
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
        T: Serialize,
    {
        match self {
            // Serialize values as an enum variant with index 0
            MachineSer::Value(t) => serializer.serialize_newtype_variant(
                "unused-enum-name",
                0,
                "unused-enum-variant",
                t,
            ),
            // Serialize fallbacks as an (empty) variant with index = fallback + 1
            MachineSer::Fallback(fallback) => serializer.serialize_unit_variant(
                "unused-enum-name",
                (fallback + 1) as u32,
                "unused-enum-variant",
            ),
        }
    }
}

impl<'de, T: Deserialize<'de>> Deserialize<'de> for MachineDe<T> {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        struct DedupeVisitor<R>(core::marker::PhantomData<R>);

        impl<'de, R: Deserialize<'de>> Visitor<'de> for DedupeVisitor<R> {
            type Value = MachineDe<R>;

            fn expecting(&self, formatter: &mut Formatter) -> Result<(), Error> {
                formatter.write_str("Element or fallback reference.")
            }

            fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
            where
                A: EnumAccess<'de>,
            {
                use serde::de::VariantAccess;

                let (variant, variant_access) = data.variant()?;
                Ok(match variant {
                    0 => MachineDe::Value(variant_access.newtype_variant()?),
                    n => MachineDe::Fallback(n - 1),
                })
            }
        }

        deserializer.deserialize_enum(
            "unused-enum-name",
            &[],
            DedupeVisitor(core::marker::PhantomData),
        )
    }
}

#[cfg(test)]
mod test {
    use alloc::borrow::Cow;
    use alloc::string::ToString;
    use serde::*;

    // Putting a Cow directly into the array doesn't borrow
    // for some reason, even with default array deserialization
    // (maybe https://github.com/serde-rs/serde/issues/2016).
    // This extra layer lets us test that borrowing works.
    #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
    struct Foo<'data>(#[serde(borrow)] Cow<'data, str>);

    #[derive(Debug, PartialEq, Serialize, Deserialize)]
    struct TestStruct<'data>(#[serde(borrow, with = "super")] [Foo<'data>; 3]);

    static STRUCT: TestStruct = TestStruct([
        Foo(Cow::Borrowed("Bar")),
        Foo(Cow::Borrowed("Batz")),
        Foo(Cow::Borrowed("Bar")),
    ]);

    #[test]
    fn test_json() {
        let json = r#"["Bar","Batz",[0]]"#;

        assert_eq!(serde_json::to_string(&STRUCT).unwrap(), json);

        assert_eq!(serde_json::from_str::<TestStruct>(json).unwrap(), STRUCT);
    }

    #[test]
    fn test_postcard() {
        #[rustfmt::skip]
        let bytes = &[
            0, // [0]
                3, // "Bar"
                    66, // B
                    97, // a
                    114, // r
            0, // [1]
                4, // "Batz"
                    66, // B
                    97, // a
                    116, // t
                    122, // z
            1, // [2] => [0]
        ];

        assert_eq!(postcard::to_allocvec(&STRUCT).unwrap(), bytes);

        let de_struct = postcard::from_bytes::<TestStruct>(bytes).unwrap();

        assert_eq!(de_struct, STRUCT);
        assert!(matches!(de_struct.0[0].0, Cow::Borrowed(_)));
    }

    #[test]
    fn test_forward_fallback() {
        assert_eq!(
            serde_json::from_str::<TestStruct>(r#"[[1], "Foo", "Batz"]"#)
                .unwrap_err()
                .to_string(),
            "Illegal forward fallback 0->1"
        );
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	1✔
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	//! A serde serialization strategy that uses `PartialEq` to reduce serialized size.
6	//!
7	//! This create can be used with Serde derive like this:
8	//!
9	//! ```rust
10	//! # #[derive(Clone, PartialEq, serde::Deserialize, serde::Serialize)]
11	//! # struct Bar(String);
12	//!
13	//! #[derive(serde::Deserialize, serde::Serialize)]
14	//! pub struct Foo {
15	//! #[serde(with = "deduplicating_array")]
16	//! data: [Bar; 12],
17	//! // ...
18	//! }
19	//! ```
20	//!
21	//! `Bar`s that are equal to a `Bar`s that appears earlier in the array will not be serialized
22	//! (instead, the index of the first occurence is serialized). Deserialization clones the first
23	//! `Bar` into all the indices where it occurs (hence `Bar` has to implement `Clone`).
24	//!
25	//! Human readable serialization represents skipped values as singleton arrays containing the
26	//! target index, e.g. the Rust array `["Foo", "Bar", "Foo"]` will serialize to JSON `["Foo", "Bar", [0]]`.
27	//!
28	//! This implies that singleton integer arrays cannot be used as array elements (they do work in Bincode,
29	//! but there's really not much point in using them).
30
31	// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
32	#![cfg_attr(not(test), no_std)]
33	#![cfg_attr(
34	not(test),
35	deny(
36	clippy::indexing_slicing,
37	clippy::unwrap_used,
38	clippy::expect_used,
39	clippy::panic,
40	clippy::exhaustive_structs,
41	clippy::exhaustive_enums,
42	missing_debug_implementations,
43	)
44	)]
45
46	extern crate alloc;
47
48	use alloc::fmt::{Error, Formatter};
49	use alloc::format;
50	use serde::de::{Deserialize, Deserializer, EnumAccess, Visitor};
51	use serde::ser::{Serialize, Serializer};
52
53	pub fn serialize<S, T, const N: usize>(array: &[T; N], serializer: S) -> Result<S::Ok, S::Error>	56✔
54	where
55	S: Serializer,
56	T: Serialize + PartialEq,
57	{
58	use serde::ser::SerializeTuple;
59
60	let human = serializer.is_human_readable();	56✔
61
62	let mut seq = serializer.serialize_tuple(N)?;	56✔
63
64	for i in 0..N {	710✔
65	#[allow(clippy::indexing_slicing)] // i, j in 0..N
66	match array.iter().take(i).position(\|item\| item == &array[i]) {	1,907✔
67	None if human => seq.serialize_element(&HumanSer::Value(&array[i]))?,	116✔
68	None => seq.serialize_element(&MachineSer::Value(&array[i]))?,	114✔
69	Some(j) if human => seq.serialize_element(&HumanSer::<T>::Fallback([j]))?,	538✔
70	Some(j) => seq.serialize_element(&MachineSer::<T>::Fallback(j))?,	593✔
71	}
72	}
73	seq.end()	56✔
74	}	56✔
75
76	pub fn deserialize<'de, D, T, const N: usize>(deserializer: D) -> Result<[T; N], D::Error>	57✔
77	where
78	D: Deserializer<'de>,
79	T: Deserialize<'de> + Clone,
80	[HumanDe<T>; N]: Deserialize<'de>,
81	[MachineDe<T>; N]: Deserialize<'de>,
82	{
83	use core::mem::MaybeUninit;
84	use serde::de::Error;
85
86	let mut array: [MaybeUninit<T>; N] = unsafe { MaybeUninit::uninit().assume_init() };	57✔
87
88	if deserializer.is_human_readable() {	57✔
89	for (i, r) in <[HumanDe<T>; N]>::deserialize(deserializer)?	6✔
90	.into_iter()
91	.enumerate()
92	{
93	match r {	4✔
94	HumanDe::Value(v) => {	2✔
95	#[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
96	array[i].write(v);	2✔
97	}	2✔
98	HumanDe::Fallback([j]) => unsafe {	2✔
99	// Fallbacks should always be to a previous value,
100	// which makes the assume_init_ref safe
101	if j >= i {	2✔
102	return Err(D::Error::custom(format!(	1✔
103	"Illegal forward fallback {i}->{j}",
104	)));
105	}
106	#[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
107	array[i].write(array[j].assume_init_ref().clone());	1✔
108	},
109	}
110	}
111	} else {
112	for (i, r) in	761✔
113	IntoIterator::into_iter(<[MachineDe<T>; N]>::deserialize(deserializer)?).enumerate()	706✔
114	{
115	match r {	651✔
116	MachineDe::Value(v) => {	114✔
117	#[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
118	array[i].write(v);	114✔
119	}	114✔
120	MachineDe::Fallback(j) => unsafe {	537✔
121	// Fallbacks should always be to a previous value,
122	// which makes the assume_init_ref safe
123	if j >= i {	537✔
124	return Err(D::Error::custom(format!(	×
125	"Illegal forward fallback {i}->{j}",
126	)));
127	}
128	#[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
129	array[i].write(array[j].assume_init_ref().clone());	537✔
130	},
131	}
132	}
133	}
134
135	// https://github.com/rust-lang/rust/issues/61956
UNCOV 136	Ok(unsafe { core::ptr::read(array.as_ptr() as *const [T; N]) })	×
137	}	169✔
138
139	#[derive(serde::Serialize)]	3✔
140	#[serde(untagged)]
141	enum HumanSer<'a, T> {
142	Value(&'a T),
143	Fallback([usize; 1]),
144	}
145
146	enum MachineSer<'a, T> {
147	Value(&'a T),
148	Fallback(usize),
149	}
150
151	// These enums are public because they are exposed in the trait bounds of
152	// deserialize. Serde only implements Deserialize<'de> on arrays up to
153	// size 32, so [Dedupe<T>;N]: Deserialize<'de> cannot be inferred from
154	// Dedupe<T>: Deserialize<'de> in the general case. See
155	// https://github.com/serde-rs/serde/issues/1937.
156	//
157	// These are not considered part of the stable API.
158
159	#[derive(serde::Deserialize, Debug)]	7✔
160	#[serde(untagged)]
161	#[doc(hidden)]
162	#[allow(clippy::exhaustive_enums)] // internal type
163	pub enum HumanDe<T> {
164	Value(T),	4✔
165	Fallback([usize; 1]),	1✔
166	}
167
168	#[doc(hidden)]
169	#[derive(Debug)]
170	#[allow(clippy::exhaustive_enums)] // internal type
171	pub enum MachineDe<T> {
172	Value(T),
173	Fallback(usize),
174	}
175
176	impl<'a, T> Serialize for MachineSer<'a, T>
177	where
178	T: Serialize,
179	{
180	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>	651✔
181	where
182	S: Serializer,
183	T: Serialize,
184	{
185	match self {	651✔
186	// Serialize values as an enum variant with index 0
187	MachineSer::Value(t) => serializer.serialize_newtype_variant(	114✔
188	"unused-enum-name",
189	0,
190	"unused-enum-variant",
191	t,
192	),
193	// Serialize fallbacks as an (empty) variant with index = fallback + 1
194	MachineSer::Fallback(fallback) => serializer.serialize_unit_variant(	537✔
195	"unused-enum-name",
196	(fallback + 1) as u32,	537✔
197	"unused-enum-variant",
198	),	537✔
199	}
200	}	651✔
201	}
202
203	impl<'de, T: Deserialize<'de>> Deserialize<'de> for MachineDe<T> {
204	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>	651✔
205	where
206	D: Deserializer<'de>,
207	{
208	struct DedupeVisitor<R>(core::marker::PhantomData<R>);
209
210	impl<'de, R: Deserialize<'de>> Visitor<'de> for DedupeVisitor<R> {
211	type Value = MachineDe<R>;
212
213	fn expecting(&self, formatter: &mut Formatter) -> Result<(), Error> {
214	formatter.write_str("Element or fallback reference.")
215	}
216
217	fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>	651✔
218	where
219	A: EnumAccess<'de>,
220	{
221	use serde::de::VariantAccess;
222
223	let (variant, variant_access) = data.variant()?;	651✔
224	Ok(match variant {	1,302✔
225	0 => MachineDe::Value(variant_access.newtype_variant()?),	114✔
226	n => MachineDe::Fallback(n - 1),	537✔
227	})
228	}	651✔
229	}
230
231	deserializer.deserialize_enum(	651✔
232	"unused-enum-name",
233	&[],
234	DedupeVisitor(core::marker::PhantomData),
235	)
236	}	651✔
237	}
238
239	#[cfg(test)]
240	mod test {
241	use alloc::borrow::Cow;
242	use alloc::string::ToString;
243	use serde::*;
244
245	// Putting a Cow directly into the array doesn't borrow
246	// for some reason, even with default array deserialization
247	// (maybe https://github.com/serde-rs/serde/issues/2016).
248	// This extra layer lets us test that borrowing works.
249	#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]	44✔
250	struct Foo<'data>(#[serde(borrow)] Cow<'data, str>);	12✔
251
252	#[derive(Debug, PartialEq, Serialize, Deserialize)]	14✔
253	struct TestStruct<'data>(#[serde(borrow, with = "super")] [Foo<'data>; 3]);	2✔
254
255	static STRUCT: TestStruct = TestStruct([
256	Foo(Cow::Borrowed("Bar")),
257	Foo(Cow::Borrowed("Batz")),
258	Foo(Cow::Borrowed("Bar")),
259	]);
260
261	#[test]
262	fn test_json() {	2✔
263	let json = r#"["Bar","Batz",[0]]"#;	1✔
264
265	assert_eq!(serde_json::to_string(&STRUCT).unwrap(), json);	1✔
266
267	assert_eq!(serde_json::from_str::<TestStruct>(json).unwrap(), STRUCT);	1✔
268	}	2✔
269
270	#[test]
271	fn test_postcard() {	2✔
272	#[rustfmt::skip]
273	let bytes = &[	1✔
274	0, // [0]
275	3, // "Bar"
276	66, // B
277	97, // a
278	114, // r
279	0, // [1]
280	4, // "Batz"
281	66, // B
282	97, // a
283	116, // t
284	122, // z
285	1, // [2] => [0]
286	];
287
288	assert_eq!(postcard::to_allocvec(&STRUCT).unwrap(), bytes);	1✔
289
290	let de_struct = postcard::from_bytes::<TestStruct>(bytes).unwrap();	1✔
291
292	assert_eq!(de_struct, STRUCT);	1✔
293	assert!(matches!(de_struct.0[0].0, Cow::Borrowed(_)));	1✔
294	}	2✔
295
296	#[test]
297	fn test_forward_fallback() {	2✔
298	assert_eq!(	1✔
299	serde_json::from_str::<TestStruct>(r#"[[1], "Foo", "Batz"]"#)	1✔
300	.unwrap_err()
301	.to_string(),
302	"Illegal forward fallback 0->1"
303	);
304	}	2✔
305	}

zbraniecki / icu4x / 9357137046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous