13958601093

Committed 19 Mar 2025 04:17PM UTC coverage: 74.164% (-1.5%) from 75.71%

Build # 13958601093

Build Type

push

github

Committed by

web-flow

Commit Message

Clean up properties docs (#6315)

Run Details

58056 of 78281 relevant lines covered (74.16%)

819371.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.44

/utils/deduplicating_array/src/lib.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! A serde serialization strategy that uses `PartialEq` to reduce serialized size.
//!
//! This create can be used with Serde derive like this:
//!
//! ```rust
//! # #[derive(Clone, PartialEq, serde::Deserialize, serde::Serialize)]
//! # struct Bar(String);
//!
//! #[derive(serde::Deserialize, serde::Serialize)]
//! pub struct Foo {
//!     #[serde(with = "deduplicating_array")]
//!     data: [Bar; 12],
//!     // ...
//! }
//! ```
//!
//! `Bar`s that are equal to a `Bar`s that appears earlier in the array will not be serialized
//! (instead, the index of the first occurence is serialized). Deserialization clones the first
//! `Bar` into all the indices where it occurs (hence `Bar` has to implement `Clone`).
//!
//! Human readable serialization represents skipped values as singleton arrays containing the
//! target index, e.g. the Rust array `["Foo", "Bar", "Foo"]` will serialize to JSON `["Foo", "Bar", [0]]`.
//!
//! This implies that singleton integer arrays cannot be used as array elements (they do work in Bincode,
//! but there's really not much point in using them).

// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
#![cfg_attr(not(any(test, doc)), no_std)]
#![cfg_attr(
    not(test),
    deny(
        clippy::indexing_slicing,
        clippy::unwrap_used,
        clippy::expect_used,
        clippy::panic,
        clippy::exhaustive_structs,
        clippy::exhaustive_enums,
        clippy::trivially_copy_pass_by_ref,
        missing_debug_implementations,
    )
)]

extern crate alloc;

use alloc::fmt::{Error, Formatter};
use alloc::format;
use serde::de::{Deserialize, Deserializer, EnumAccess, Visitor};
use serde::ser::{Serialize, Serializer};

pub fn serialize<S, T, const N: usize>(array: &[T; N], serializer: S) -> Result<S::Ok, S::Error>
where
    S: Serializer,
    T: Serialize + PartialEq,
{
    use serde::ser::SerializeTuple;

    let human = serializer.is_human_readable();

    let mut seq = serializer.serialize_tuple(N)?;

    for i in 0..N {
        #[allow(clippy::indexing_slicing)] // i, j in 0..N
        match array.iter().take(i).position(|item| item == &array[i]) {
            None if human => seq.serialize_element(&HumanSer::Value(&array[i]))?,
            None => seq.serialize_element(&MachineSer::Value(&array[i]))?,
            Some(j) if human => seq.serialize_element(&HumanSer::<T>::Fallback([j]))?,
            Some(j) => seq.serialize_element(&MachineSer::<T>::Fallback(j))?,
        }
    }
    seq.end()
}

pub fn deserialize<'de, D, T, const N: usize>(deserializer: D) -> Result<[T; N], D::Error>
where
    D: Deserializer<'de>,
    T: Deserialize<'de> + Clone,
    [HumanDe<T>; N]: Deserialize<'de>,
    [MachineDe<T>; N]: Deserialize<'de>,
{
    use core::mem::MaybeUninit;
    use serde::de::Error;

    let mut array: [MaybeUninit<T>; N] = unsafe { MaybeUninit::uninit().assume_init() };

    if deserializer.is_human_readable() {
        for (i, r) in <[HumanDe<T>; N]>::deserialize(deserializer)?
            .into_iter()
            .enumerate()
        {
            match r {
                HumanDe::Value(v) => {
                    #[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
                    array[i].write(v);
                }
                HumanDe::Fallback([j]) => unsafe {
                    // Fallbacks should always be to a previous value,
                    // which makes the assume_init_ref safe
                    if j >= i {
                        return Err(D::Error::custom(format!(
                            "Illegal forward fallback {i}->{j}",
                        )));
                    }
                    #[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
                    array[i].write(array[j].assume_init_ref().clone());
                },
            }
        }
    } else {
        for (i, r) in
            IntoIterator::into_iter(<[MachineDe<T>; N]>::deserialize(deserializer)?).enumerate()
        {
            match r {
                MachineDe::Value(v) => {
                    #[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
                    array[i].write(v);
                }
                MachineDe::Fallback(j) => unsafe {
                    // Fallbacks should always be to a previous value,
                    // which makes the assume_init_ref safe
                    if j >= i {
                        return Err(D::Error::custom(format!(
                            "Illegal forward fallback {i}->{j}",
                        )));
                    }
                    #[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
                    array[i].write(array[j].assume_init_ref().clone());
                },
            }
        }
    }

    // https://github.com/rust-lang/rust/issues/61956
    Ok(unsafe { core::ptr::read(array.as_ptr() as *const [T; N]) })
}

#[derive(serde::Serialize)]
#[serde(untagged)]
enum HumanSer<'a, T> {
    Value(&'a T),
    Fallback([usize; 1]),
}

enum MachineSer<'a, T> {
    Value(&'a T),
    Fallback(usize),
}

// These enums are public because they are exposed in the trait bounds of
// deserialize. Serde only implements Deserialize<'de> on arrays up to
// size 32, so [Dedupe<T>;N]: Deserialize<'de> cannot be inferred from
// Dedupe<T>: Deserialize<'de> in the general case. See
// https://github.com/serde-rs/serde/issues/1937.
//
// These are not considered part of the stable API.

#[derive(serde::Deserialize, Debug)]
#[serde(untagged)]
#[doc(hidden)] // appears in public bound
#[allow(clippy::exhaustive_enums)] // internal type
pub enum HumanDe<T> {
    Value(T),
    Fallback([usize; 1]),
}

#[doc(hidden)] // appears in public bound
#[derive(Debug)]
#[allow(clippy::exhaustive_enums)] // internal type
pub enum MachineDe<T> {
    Value(T),
    Fallback(usize),
}

impl<T> Serialize for MachineSer<'_, T>
where
    T: Serialize,
{
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
        T: Serialize,
    {
        match self {
            // Serialize values as an enum variant with index 0
            MachineSer::Value(t) => serializer.serialize_newtype_variant(
                "unused-enum-name",
                0,
                "unused-enum-variant",
                t,
            ),
            // Serialize fallbacks as an (empty) variant with index = fallback + 1
            MachineSer::Fallback(fallback) => serializer.serialize_unit_variant(
                "unused-enum-name",
                (fallback + 1) as u32,
                "unused-enum-variant",
            ),
        }
    }
}

impl<'de, T: Deserialize<'de>> Deserialize<'de> for MachineDe<T> {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        struct DedupeVisitor<R>(core::marker::PhantomData<R>);

        impl<'de, R: Deserialize<'de>> Visitor<'de> for DedupeVisitor<R> {
            type Value = MachineDe<R>;

            fn expecting(&self, formatter: &mut Formatter) -> Result<(), Error> {
                formatter.write_str("Element or fallback reference.")
            }

            fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
            where
                A: EnumAccess<'de>,
            {
                use serde::de::VariantAccess;

                let (variant, variant_access) = data.variant()?;
                Ok(match variant {
                    0 => MachineDe::Value(variant_access.newtype_variant()?),
                    n => MachineDe::Fallback(n - 1),
                })
            }
        }

        deserializer.deserialize_enum(
            "unused-enum-name",
            &[],
            DedupeVisitor(core::marker::PhantomData),
        )
    }
}

#[cfg(test)]
mod test {
    use alloc::borrow::Cow;
    use alloc::string::ToString;
    use serde::*;

    // Putting a Cow directly into the array doesn't borrow
    // for some reason, even with default array deserialization
    // (maybe https://github.com/serde-rs/serde/issues/2016).
    // This extra layer lets us test that borrowing works.
    #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
    struct Foo<'data>(#[serde(borrow)] Cow<'data, str>);

    #[derive(Debug, PartialEq, Serialize, Deserialize)]
    struct TestStruct<'data>(#[serde(borrow, with = "super")] [Foo<'data>; 3]);

    static STRUCT: TestStruct = TestStruct([
        Foo(Cow::Borrowed("Bar")),
        Foo(Cow::Borrowed("Batz")),
        Foo(Cow::Borrowed("Bar")),
    ]);

    #[test]
    fn test_json() {
        let json = r#"["Bar","Batz",[0]]"#;

        assert_eq!(serde_json::to_string(&STRUCT).unwrap(), json);

        assert_eq!(serde_json::from_str::<TestStruct>(json).unwrap(), STRUCT);
    }

    #[test]
    fn test_postcard() {
        #[rustfmt::skip]
        let bytes = &[
            0, // [0]
                3, // "Bar"
                    66, // B
                    97, // a
                    114, // r
            0, // [1]
                4, // "Batz"
                    66, // B
                    97, // a
                    116, // t
                    122, // z
            1, // [2] => [0]
        ];

        assert_eq!(postcard::to_allocvec(&STRUCT).unwrap(), bytes);

        let de_struct = postcard::from_bytes::<TestStruct>(bytes).unwrap();

        assert_eq!(de_struct, STRUCT);
        assert!(matches!(de_struct.0[0].0, Cow::Borrowed(_)));
    }

    #[test]
    fn test_forward_fallback() {
        assert_eq!(
            serde_json::from_str::<TestStruct>(r#"[[1], "Foo", "Batz"]"#)
                .unwrap_err()
                .to_string(),
            "Illegal forward fallback 0->1"
        );
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	1✔
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	//! A serde serialization strategy that uses `PartialEq` to reduce serialized size.
6	//!
7	//! This create can be used with Serde derive like this:
8	//!
9	//! ```rust
10	//! # #[derive(Clone, PartialEq, serde::Deserialize, serde::Serialize)]
11	//! # struct Bar(String);
12	//!
13	//! #[derive(serde::Deserialize, serde::Serialize)]
14	//! pub struct Foo {
15	//! #[serde(with = "deduplicating_array")]
16	//! data: [Bar; 12],
17	//! // ...
18	//! }
19	//! ```
20	//!
21	//! `Bar`s that are equal to a `Bar`s that appears earlier in the array will not be serialized
22	//! (instead, the index of the first occurence is serialized). Deserialization clones the first
23	//! `Bar` into all the indices where it occurs (hence `Bar` has to implement `Clone`).
24	//!
25	//! Human readable serialization represents skipped values as singleton arrays containing the
26	//! target index, e.g. the Rust array `["Foo", "Bar", "Foo"]` will serialize to JSON `["Foo", "Bar", [0]]`.
27	//!
28	//! This implies that singleton integer arrays cannot be used as array elements (they do work in Bincode,
29	//! but there's really not much point in using them).
30
31	// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
32	#![cfg_attr(not(any(test, doc)), no_std)]
33	#![cfg_attr(
34	not(test),
35	deny(
36	clippy::indexing_slicing,
37	clippy::unwrap_used,
38	clippy::expect_used,
39	clippy::panic,
40	clippy::exhaustive_structs,
41	clippy::exhaustive_enums,
42	clippy::trivially_copy_pass_by_ref,
43	missing_debug_implementations,
44	)
45	)]
46
47	extern crate alloc;
48
49	use alloc::fmt::{Error, Formatter};
50	use alloc::format;
51	use serde::de::{Deserialize, Deserializer, EnumAccess, Visitor};
52	use serde::ser::{Serialize, Serializer};
53
54	pub fn serialize<S, T, const N: usize>(array: &[T; N], serializer: S) -> Result<S::Ok, S::Error>	2✔
55	where
56	S: Serializer,
57	T: Serialize + PartialEq,
58	{
59	use serde::ser::SerializeTuple;
60
61	let human = serializer.is_human_readable();	2✔
62
63	let mut seq = serializer.serialize_tuple(N)?;	2✔
64
65	for i in 0..N {	8✔
66	#[allow(clippy::indexing_slicing)] // i, j in 0..N
67	match array.iter().take(i).position(\|item\| item == &array[i]) {	10✔
68	None if human => seq.serialize_element(&HumanSer::Value(&array[i]))?,	4✔
69	None => seq.serialize_element(&MachineSer::Value(&array[i]))?,	2✔
70	Some(j) if human => seq.serialize_element(&HumanSer::<T>::Fallback([j]))?,	2✔
71	Some(j) => seq.serialize_element(&MachineSer::<T>::Fallback(j))?,	3✔
72	}
73	}
74	seq.end()	2✔
75	}	2✔
76
77	pub fn deserialize<'de, D, T, const N: usize>(deserializer: D) -> Result<[T; N], D::Error>	3✔
78	where
79	D: Deserializer<'de>,
80	T: Deserialize<'de> + Clone,
81	[HumanDe<T>; N]: Deserialize<'de>,
82	[MachineDe<T>; N]: Deserialize<'de>,
83	{
84	use core::mem::MaybeUninit;
85	use serde::de::Error;
86
87	let mut array: [MaybeUninit<T>; N] = unsafe { MaybeUninit::uninit().assume_init() };	3✔
88
89	if deserializer.is_human_readable() {	3✔
90	for (i, r) in <[HumanDe<T>; N]>::deserialize(deserializer)?	5✔
91	.into_iter()
92	.enumerate()
93	{
94	match r {	4✔
95	HumanDe::Value(v) => {	2✔
96	#[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
97	array[i].write(v);	2✔
98	}	2✔
99	HumanDe::Fallback([j]) => unsafe {	2✔
100	// Fallbacks should always be to a previous value,
101	// which makes the assume_init_ref safe
102	if j >= i {	2✔
103	return Err(D::Error::custom(format!(	1✔
104	"Illegal forward fallback {i}->{j}",
105	)));
106	}
107	#[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
108	array[i].write(array[j].assume_init_ref().clone());	1✔
109	},
110	}
111	}	2✔
112	} else {
113	for (i, r) in	4✔
114	IntoIterator::into_iter(<[MachineDe<T>; N]>::deserialize(deserializer)?).enumerate()	4✔
115	{
116	match r {	3✔
117	MachineDe::Value(v) => {	2✔
118	#[allow(clippy::indexing_slicing)] // i in 0..N by enumerate
119	array[i].write(v);	2✔
120	}	2✔
121	MachineDe::Fallback(j) => unsafe {	1✔
122	// Fallbacks should always be to a previous value,
123	// which makes the assume_init_ref safe
124	if j >= i {	1✔
125	return Err(D::Error::custom(format!(	×
126	"Illegal forward fallback {i}->{j}",
127	)));
128	}
129	#[allow(clippy::indexing_slicing)] // j < i in 0..N by enumerate
130	array[i].write(array[j].assume_init_ref().clone());	1✔
131	},
132	}
133	}	1✔
134	}
135
136	// https://github.com/rust-lang/rust/issues/61956
137	Ok(unsafe { core::ptr::read(array.as_ptr() as *const [T; N]) })	×
138	}	7✔
139
140	#[derive(serde::Serialize)]	3✔
141	#[serde(untagged)]
142	enum HumanSer<'a, T> {
143	Value(&'a T),
144	Fallback([usize; 1]),
145	}
146
147	enum MachineSer<'a, T> {
148	Value(&'a T),
149	Fallback(usize),
150	}
151
152	// These enums are public because they are exposed in the trait bounds of
153	// deserialize. Serde only implements Deserialize<'de> on arrays up to
154	// size 32, so [Dedupe<T>;N]: Deserialize<'de> cannot be inferred from
155	// Dedupe<T>: Deserialize<'de> in the general case. See
156	// https://github.com/serde-rs/serde/issues/1937.
157	//
158	// These are not considered part of the stable API.
159
160	#[derive(serde::Deserialize, Debug)]	6✔
161	#[serde(untagged)]
162	#[doc(hidden)] // appears in public bound
163	#[allow(clippy::exhaustive_enums)] // internal type
164	pub enum HumanDe<T> {
165	Value(T),	4✔
166	Fallback([usize; 1]),	2✔
167	}
168
169	#[doc(hidden)] // appears in public bound
170	#[derive(Debug)]
171	#[allow(clippy::exhaustive_enums)] // internal type
172	pub enum MachineDe<T> {
173	Value(T),
174	Fallback(usize),
175	}
176
177	impl<T> Serialize for MachineSer<'_, T>
178	where
179	T: Serialize,
180	{
181	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>	3✔
182	where
183	S: Serializer,
184	T: Serialize,
185	{
186	match self {	3✔
187	// Serialize values as an enum variant with index 0
188	MachineSer::Value(t) => serializer.serialize_newtype_variant(	2✔
189	"unused-enum-name",
190	0,
191	"unused-enum-variant",
192	t,
193	),
194	// Serialize fallbacks as an (empty) variant with index = fallback + 1
195	MachineSer::Fallback(fallback) => serializer.serialize_unit_variant(	1✔
196	"unused-enum-name",
197	(fallback + 1) as u32,	1✔
198	"unused-enum-variant",
199	),	1✔
200	}
201	}	3✔
202	}
203
204	impl<'de, T: Deserialize<'de>> Deserialize<'de> for MachineDe<T> {
205	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>	3✔
206	where
207	D: Deserializer<'de>,
208	{
209	struct DedupeVisitor<R>(core::marker::PhantomData<R>);
210
211	impl<'de, R: Deserialize<'de>> Visitor<'de> for DedupeVisitor<R> {
212	type Value = MachineDe<R>;
213
214	fn expecting(&self, formatter: &mut Formatter) -> Result<(), Error> {
215	formatter.write_str("Element or fallback reference.")
216	}
217
218	fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>	3✔
219	where
220	A: EnumAccess<'de>,
221	{
222	use serde::de::VariantAccess;
223
224	let (variant, variant_access) = data.variant()?;	3✔
225	Ok(match variant {	6✔
226	0 => MachineDe::Value(variant_access.newtype_variant()?),	2✔
227	n => MachineDe::Fallback(n - 1),	1✔
228	})
229	}	3✔
230	}
231
232	deserializer.deserialize_enum(	3✔
233	"unused-enum-name",
234	&[],
235	DedupeVisitor(core::marker::PhantomData),
236	)
237	}	3✔
238	}
239
240	#[cfg(test)]
241	mod test {
242	use alloc::borrow::Cow;
243	use alloc::string::ToString;
244	use serde::*;
245
246	// Putting a Cow directly into the array doesn't borrow
247	// for some reason, even with default array deserialization
248	// (maybe https://github.com/serde-rs/serde/issues/2016).
249	// This extra layer lets us test that borrowing works.
250	#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]	44✔
251	struct Foo<'data>(#[serde(borrow)] Cow<'data, str>);	12✔
252
253	#[derive(Debug, PartialEq, Serialize, Deserialize)]	17✔
254	struct TestStruct<'data>(#[serde(borrow, with = "super")] [Foo<'data>; 3]);	7✔
255
256	static STRUCT: TestStruct = TestStruct([
257	Foo(Cow::Borrowed("Bar")),
258	Foo(Cow::Borrowed("Batz")),
259	Foo(Cow::Borrowed("Bar")),
260	]);
261
262	#[test]
263	fn test_json() {	2✔
264	let json = r#"["Bar","Batz",[0]]"#;	1✔
265
266	assert_eq!(serde_json::to_string(&STRUCT).unwrap(), json);	2✔
267
268	assert_eq!(serde_json::from_str::<TestStruct>(json).unwrap(), STRUCT);	2✔
269	}	2✔
270
271	#[test]
272	fn test_postcard() {	2✔
273	#[rustfmt::skip]
274	let bytes = &[	1✔
275	0, // [0]
276	3, // "Bar"
277	66, // B
278	97, // a
279	114, // r
280	0, // [1]
281	4, // "Batz"
282	66, // B
283	97, // a
284	116, // t
285	122, // z
286	1, // [2] => [0]
287	];
288
289	assert_eq!(postcard::to_allocvec(&STRUCT).unwrap(), bytes);	2✔
290
291	let de_struct = postcard::from_bytes::<TestStruct>(bytes).unwrap();	1✔
292
293	assert_eq!(de_struct, STRUCT);	1✔
294	assert!(matches!(de_struct.0[0].0, Cow::Borrowed(_)));	1✔
295	}	2✔
296
297	#[test]
298	fn test_forward_fallback() {	2✔
299	assert_eq!(	2✔
300	serde_json::from_str::<TestStruct>(r#"[[1], "Foo", "Batz"]"#)	1✔
301	.unwrap_err()
302	.to_string(),
303	"Illegal forward fallback 0->1"
304	);
305	}	2✔
306	}

zbraniecki / icu4x / 13958601093

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous