13958601093

Committed 19 Mar 2025 04:17PM UTC coverage: 74.164% (-1.5%) from 75.71%

Build # 13958601093

Build Type

push

github

Committed by

web-flow

Commit Message

Clean up properties docs (#6315)

Run Details

58056 of 78281 relevant lines covered (74.16%)

819371.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.0

/provider/source/src/normalizer/mod.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! This module contains provider implementations backed by TOML files
//! exported from ICU.

use crate::SourceDataProvider;
use icu::collections::char16trie::Char16Trie;
use icu::collections::codepointtrie::CodePointTrie;
use icu::normalizer::provider::*;
use icu_provider::prelude::*;
use std::collections::HashSet;
use std::convert::TryFrom;
use zerovec::ZeroVec;

mod normalizer_serde;

macro_rules! normalization_provider {
    ($marker:ident, $serde_struct:ident, $file_name:literal, $conversion:expr, $toml_data:ident) => {
        use icu::normalizer::provider::$marker;

        impl DataProvider<$marker> for SourceDataProvider {
            fn load(&self, req: DataRequest) -> Result<DataResponse<$marker>, DataError> {
                self.check_req::<$marker>(req)?;
                let $toml_data: &normalizer_serde::$serde_struct =
                    self.icuexport()?.read_and_parse_toml(&format!(
                        "norm/{}/{}.toml",
                        self.trie_type(),
                        $file_name
                    ))?;

                $conversion
            }
        }

        impl crate::IterableDataProviderCached<$marker> for SourceDataProvider {
            fn iter_ids_cached(&self) -> Result<HashSet<DataIdentifierCow<'static>>, DataError> {
                Ok(HashSet::from_iter([Default::default()]))
            }
        }
    };
}

macro_rules! normalization_data_provider {
    ($marker:ident, $file_name:literal) => {
        normalization_provider!(
            $marker,
            DecompositionData,
            $file_name,
            {
                let trie = CodePointTrie::<u32>::try_from(&toml_data.trie)
                    .map_err(|e| DataError::custom("trie conversion").with_display_context(&e))?;

                Ok(DataResponse {
                    metadata: Default::default(),
                    payload: DataPayload::from_owned(DecompositionData {
                        trie,
                        passthrough_cap: toml_data.cap,
                    }),
                })
            },
            toml_data // simply matches the identifier in the above block
        );
    };
}

macro_rules! normalization_tables_provider {
    ($marker:ident, $file_name:literal) => {
        normalization_provider!(
            $marker,
            DecompositionTables,
            $file_name,
            {
                let scalars24 = toml_data
                    .scalars32
                    .iter()
                    .map(|&u| {
                        u.try_into()
                            .map_err(|_| DataError::custom("scalars24 conversion"))
                    })
                    .collect::<Result<Vec<char>, DataError>>()?;
                Ok(DataResponse {
                    metadata: Default::default(),
                    payload: DataPayload::from_owned(DecompositionTables {
                        scalars16: ZeroVec::alloc_from_slice(&toml_data.scalars16),
                        scalars24: ZeroVec::alloc_from_slice(&scalars24),
                    }),
                })
            },
            toml_data // simply matches the identifier in the above block
        );
    };
}

macro_rules! normalization_canonical_compositions_provider {
    ($marker:ident, $file_name:literal) => {
        normalization_provider!(
            $marker,
            CanonicalCompositions,
            $file_name,
            {
                Ok(DataResponse {
                    metadata: Default::default(),
                    payload: DataPayload::from_owned(CanonicalCompositions {
                        canonical_compositions: Char16Trie::new(ZeroVec::alloc_from_slice(
                            &toml_data.compositions,
                        )),
                    }),
                })
            },
            toml_data // simply matches the identifier in the above block
        );
    };
}

macro_rules! normalization_non_recursive_decomposition_supplement_provider {
    ($marker:ident, $file_name:literal) => {
        normalization_provider!(
            $marker,
            NonRecursiveDecompositionSupplement,
            $file_name,
            {
                let trie = CodePointTrie::<u32>::try_from(&toml_data.trie)
                    .map_err(|e| DataError::custom("trie conversion").with_display_context(&e))?;
                let scalars24 = toml_data
                    .scalars32
                    .iter()
                    .map(|&u| {
                        u.try_into()
                            .map_err(|_| DataError::custom("scalars24 conversion"))
                    })
                    .collect::<Result<Vec<char>, DataError>>()?;

                Ok(DataResponse {
                    metadata: Default::default(),
                    payload: DataPayload::from_owned(NonRecursiveDecompositionSupplement {
                        trie,
                        scalars24: ZeroVec::alloc_from_slice(&scalars24),
                    }),
                })
            },
            toml_data // simply matches the identifier in the above block
        );
    };
}

normalization_data_provider!(NormalizerNfdDataV1, "nfd");

normalization_data_provider!(NormalizerNfkdDataV1, "nfkd");

normalization_data_provider!(NormalizerUts46DataV1, "uts46d");

normalization_tables_provider!(NormalizerNfdTablesV1, "nfdex");

normalization_tables_provider!(NormalizerNfkdTablesV1, "nfkdex");

// No uts46dex, because that data is also in nfkdex.

normalization_canonical_compositions_provider!(NormalizerNfcV1, "compositions");

normalization_non_recursive_decomposition_supplement_provider!(
    NormalizerNfdSupplementV1,
    "decompositionex"
);

1	// This file is part of ICU4X. For terms of use, please see the file
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	//! This module contains provider implementations backed by TOML files
6	//! exported from ICU.
7
8	use crate::SourceDataProvider;
9	use icu::collections::char16trie::Char16Trie;
10	use icu::collections::codepointtrie::CodePointTrie;
11	use icu::normalizer::provider::*;
12	use icu_provider::prelude::*;
13	use std::collections::HashSet;
14	use std::convert::TryFrom;
15	use zerovec::ZeroVec;
16
17	mod normalizer_serde;
18
19	macro_rules! normalization_provider {
20	($marker:ident, $serde_struct:ident, $file_name:literal, $conversion:expr, $toml_data:ident) => {
21	use icu::normalizer::provider::$marker;
22
23	impl DataProvider<$marker> for SourceDataProvider {
24	fn load(&self, req: DataRequest) -> Result<DataResponse<$marker>, DataError> {	7✔
25	self.check_req::<$marker>(req)?;	7✔
26	let $toml_data: &normalizer_serde::$serde_struct =
27	self.icuexport()?.read_and_parse_toml(&format!(	7✔
28	"norm/{}/{}.toml",
29	self.trie_type(),	7✔
30	$file_name
31	))?;	7✔
32
33	$conversion
34	}	7✔
35	}
36
37	impl crate::IterableDataProviderCached<$marker> for SourceDataProvider {
38	fn iter_ids_cached(&self) -> Result<HashSet<DataIdentifierCow<'static>>, DataError> {	×
39	Ok(HashSet::from_iter([Default::default()]))	×
40	}	×
41	}
42	};
43	}
44
45	macro_rules! normalization_data_provider {
46	($marker:ident, $file_name:literal) => {
47	normalization_provider!(
48	$marker,
49	DecompositionData,
50	$file_name,
51	{
52	let trie = CodePointTrie::<u32>::try_from(&toml_data.trie)	3✔
53	.map_err(\|e\| DataError::custom("trie conversion").with_display_context(&e))?;	×
54
55	Ok(DataResponse {	3✔
56	metadata: Default::default(),	3✔
57	payload: DataPayload::from_owned(DecompositionData {	3✔
58	trie,	3✔
59	passthrough_cap: toml_data.cap,	3✔
60	}),
61	})
62	},	3✔
63	toml_data // simply matches the identifier in the above block
64	);
65	};
66	}
67
68	macro_rules! normalization_tables_provider {
69	($marker:ident, $file_name:literal) => {
70	normalization_provider!(
71	$marker,
72	DecompositionTables,
73	$file_name,
74	{
75	let scalars24 = toml_data	2✔
76	.scalars32
77	.iter()
78	.map(\|&u\| {	483✔
79	u.try_into()	483✔
80	.map_err(\|_\| DataError::custom("scalars24 conversion"))	×
81	})	483✔
82	.collect::<Result<Vec<char>, DataError>>()?;	×
83	Ok(DataResponse {	2✔
84	metadata: Default::default(),	2✔
85	payload: DataPayload::from_owned(DecompositionTables {	2✔
86	scalars16: ZeroVec::alloc_from_slice(&toml_data.scalars16),	2✔
87	scalars24: ZeroVec::alloc_from_slice(&scalars24),	2✔
88	}),	×
89	})
90	},	2✔
91	toml_data // simply matches the identifier in the above block
92	);
93	};
94	}
95
96	macro_rules! normalization_canonical_compositions_provider {
97	($marker:ident, $file_name:literal) => {
98	normalization_provider!(
99	$marker,
100	CanonicalCompositions,
101	$file_name,
102	{
103	Ok(DataResponse {	1✔
104	metadata: Default::default(),	1✔
105	payload: DataPayload::from_owned(CanonicalCompositions {	1✔
106	canonical_compositions: Char16Trie::new(ZeroVec::alloc_from_slice(	1✔
107	&toml_data.compositions,	1✔
108	)),
109	}),
110	})
111	},
112	toml_data // simply matches the identifier in the above block
113	);
114	};
115	}
116
117	macro_rules! normalization_non_recursive_decomposition_supplement_provider {
118	($marker:ident, $file_name:literal) => {
119	normalization_provider!(
120	$marker,
121	NonRecursiveDecompositionSupplement,
122	$file_name,
123	{
124	let trie = CodePointTrie::<u32>::try_from(&toml_data.trie)	1✔
125	.map_err(\|e\| DataError::custom("trie conversion").with_display_context(&e))?;	×
126	let scalars24 = toml_data	1✔
127	.scalars32
128	.iter()
129	.map(\|&u\| {	26✔
130	u.try_into()	26✔
131	.map_err(\|_\| DataError::custom("scalars24 conversion"))	×
132	})	26✔
133	.collect::<Result<Vec<char>, DataError>>()?;	×
134
135	Ok(DataResponse {	1✔
136	metadata: Default::default(),	1✔
137	payload: DataPayload::from_owned(NonRecursiveDecompositionSupplement {	1✔
138	trie,	1✔
139	scalars24: ZeroVec::alloc_from_slice(&scalars24),	1✔
140	}),	×
141	})
142	},	1✔
143	toml_data // simply matches the identifier in the above block
144	);
145	};
146	}
147
148	normalization_data_provider!(NormalizerNfdDataV1, "nfd");
149
150	normalization_data_provider!(NormalizerNfkdDataV1, "nfkd");
151
152	normalization_data_provider!(NormalizerUts46DataV1, "uts46d");
153
154	normalization_tables_provider!(NormalizerNfdTablesV1, "nfdex");
155
156	normalization_tables_provider!(NormalizerNfkdTablesV1, "nfkdex");
157
158	// No uts46dex, because that data is also in nfkdex.
159
160	normalization_canonical_compositions_provider!(NormalizerNfcV1, "compositions");
161
162	normalization_non_recursive_decomposition_supplement_provider!(
163	NormalizerNfdSupplementV1,
164	"decompositionex"
165	);

zbraniecki / icu4x / 13958601093

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous