9357137046

Committed 03 Jun 2024 08:51PM UTC coverage: 75.121% (-1.1%) from 76.254%

Build # 9357137046

Build Type

push

github

Committed by

web-flow

Commit Message

Switch locid Value to use Subtag (#4941)

This is part of #1833 switching Value API to use Subtag.

Run Details

61 of 71 new or added lines in 11 files covered. (85.92%)

3224 existing lines in 178 files now uncovered.

52958 of 70497 relevant lines covered (75.12%)

572757.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.08

/tools/bakeddata-scripts/src/main.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

extern crate icu_datagen;

use icu_datagen::baked_exporter;
use icu_datagen::fs_exporter;
use icu_datagen::fs_exporter::serializers::AbstractSerializer;
use icu_datagen::prelude::*;
use icu_provider::datagen::*;
use icu_provider::prelude::*;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::sync::Mutex;

const REPO_VERSION: &str = env!("CARGO_PKG_VERSION");
const EXPERIMENTAL_VERSION: &str = "0.1.0";
const COMPONENTS: &[(&str, &[DataKey], &str)] = &[
    ("calendar", icu::calendar::provider::KEYS, REPO_VERSION),
    ("casemap", icu::casemap::provider::KEYS, REPO_VERSION),
    ("collator", icu::collator::provider::KEYS, REPO_VERSION),
    ("datetime", icu::datetime::provider::KEYS, REPO_VERSION),
    ("decimal", icu::decimal::provider::KEYS, REPO_VERSION),
    ("list", icu::list::provider::KEYS, REPO_VERSION),
    ("locale", icu::locale::provider::KEYS, REPO_VERSION),
    ("normalizer", icu::normalizer::provider::KEYS, REPO_VERSION),
    ("plurals", icu::plurals::provider::KEYS, REPO_VERSION),
    ("properties", icu::properties::provider::KEYS, REPO_VERSION),
    ("segmenter", icu::segmenter::provider::KEYS, REPO_VERSION),
    ("timezone", icu::timezone::provider::KEYS, REPO_VERSION),
    (
        "experimental",
        icu::experimental::provider::KEYS,
        EXPERIMENTAL_VERSION,
    ),
];

fn main() {
    simple_logger::SimpleLogger::new()
        .env()
        .with_level(log::LevelFilter::Info)
        .init()
        .unwrap();

    let args = std::env::args().skip(1).collect::<Vec<_>>();

    let components = if args.is_empty() {
        COMPONENTS
            .iter()
            .map(|(krate, keys, version)| (krate.to_string(), *keys, *version))
            .collect::<Vec<_>>()
    } else {
        let map = std::collections::HashMap::<&str, (&'static [DataKey], &'static str)>::from_iter(
            COMPONENTS
                .iter()
                .map(|(krate, keys, version)| (*krate, (*keys, *version))),
        );
        args.into_iter()
            .filter_map(|krate| {
                map.get(krate.as_str())
                    .map(|(keys, version)| (krate, *keys, *version))
            })
            .collect()
    };

    let source = DatagenProvider::new_latest_tested();

    let driver = DatagenDriver::new()
        .with_locales_and_fallback(
            source
                .locales_for_coverage_levels([
                    CoverageLevel::Modern,
                    CoverageLevel::Moderate,
                    CoverageLevel::Basic,
                ])
                .unwrap()
                .into_iter()
                .map(LocaleFamily::with_descendants),
            Default::default(),
        )
        .with_recommended_segmenter_models();

    let mut options = baked_exporter::Options::default();
    options.overwrite = true;
    options.pretty = true;

    for (component, keys, version) in &components {
        let path = Path::new("provider/baked").join(component);

        let _ = std::fs::remove_dir_all(&path);
        for dir in ["", "src", "data"] {
            std::fs::create_dir(&path.join(dir)).unwrap();
        }
        for (file, template) in [
            ("build.rs", include_str!("../template/build.rs.template")),
            (
                "Cargo.toml",
                include_str!("../template/Cargo.toml.template"),
            ),
            ("LICENSE", include_str!("../LICENSE")),
            ("README.md", include_str!("../template/README.md.template")),
            (
                "src/lib.rs",
                include_str!("../template/src/lib.rs.template"),
            ),
        ] {
            std::fs::write(
                path.join(file),
                template
                    .replace("_component_", component)
                    .replace("_version_", version)
                    .replace("_cldr_tag_", DatagenProvider::LATEST_TESTED_CLDR_TAG)
                    .replace(
                        "_icuexport_tag_",
                        DatagenProvider::LATEST_TESTED_ICUEXPORT_TAG,
                    )
                    .replace(
                        "_segmenter_lstm_tag_",
                        DatagenProvider::LATEST_TESTED_SEGMENTER_LSTM_TAG,
                    ),
            )
            .unwrap();
        }

        let baked_exporter =
            baked_exporter::BakedExporter::new(path.join("data"), options).unwrap();
        let fingerprinter = PostcardFingerprintExporter {
            size_hash: Default::default(),
            fingerprints: crlify::BufWriterWithLineEndingFix::new(
                File::create(path.join("fingerprints.csv")).unwrap(),
            ),
        };

        driver
            .clone()
            .with_keys(keys.iter().copied())
            .export(
                &source,
                MultiExporter::new(vec![Box::new(baked_exporter), Box::new(fingerprinter)]),
            )
            .unwrap();

        for file in ["data/any.rs", "data/mod.rs"] {
            std::fs::remove_file(path.join(file)).unwrap();
        }
    }
}

struct PostcardFingerprintExporter<F> {
    size_hash: Mutex<BTreeMap<(DataKey, String), (usize, u64)>>,
    fingerprints: F,
}

impl<F: Write + Send + Sync> DataExporter for PostcardFingerprintExporter<F> {
    fn put_payload(
        &self,
        key: DataKey,
        locale: &DataLocale,
        key_attributes: &DataKeyAttributes,
        payload_before: &DataPayload<ExportMarker>,
    ) -> Result<(), DataError> {
        let mut serialized = vec![];

        fs_exporter::serializers::Postcard::new(Default::default())
            .serialize(payload_before, &mut serialized)?;

        let size = serialized.len();

        // We're using SipHash, which is deprecated, but we want a stable hasher
        // (we're fine with it not being cryptographically secure since we're just using it to track diffs)
        #[allow(deprecated)]
        use std::hash::{Hash, Hasher, SipHasher};
        #[allow(deprecated)]
        let mut hasher = SipHasher::new();
        serialized.iter().for_each(|b| b.hash(&mut hasher));
        let hash = hasher.finish();

        self.size_hash.lock().expect("poison").insert(
            (
                key,
                DataRequest {
                    locale,
                    key_attributes,
                    ..Default::default()
                }
                .legacy_encode(),
            ),
            (size, hash),
        );

        Ok(())
    }

    fn flush(&self, _key: DataKey) -> Result<(), DataError> {
        Ok(())
    }

    fn flush_with_built_in_fallback(
        &self,
        _key: DataKey,
        _fallback_mode: BuiltInFallbackMode,
    ) -> Result<(), DataError> {
        Ok(())
    }

    fn close(&mut self) -> Result<(), DataError> {
        for ((key, req), (size, hash)) in self.size_hash.get_mut().expect("poison") {
            writeln!(&mut self.fingerprints, "{key}, {req}, {size}B, {hash:x}")?;
        }
        Ok(())
    }
    fn supports_built_in_fallback(&self) -> bool {
        true
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file	1✔
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	extern crate icu_datagen;
6
7	use icu_datagen::baked_exporter;
8	use icu_datagen::fs_exporter;
9	use icu_datagen::fs_exporter::serializers::AbstractSerializer;
10	use icu_datagen::prelude::*;
11	use icu_provider::datagen::*;
12	use icu_provider::prelude::*;
13	use std::collections::BTreeMap;
14	use std::fs::File;
15	use std::io::Write;
16	use std::path::Path;
17	use std::sync::Mutex;
18
19	const REPO_VERSION: &str = env!("CARGO_PKG_VERSION");
20	const EXPERIMENTAL_VERSION: &str = "0.1.0";
21	const COMPONENTS: &[(&str, &[DataKey], &str)] = &[
22	("calendar", icu::calendar::provider::KEYS, REPO_VERSION),
23	("casemap", icu::casemap::provider::KEYS, REPO_VERSION),
24	("collator", icu::collator::provider::KEYS, REPO_VERSION),
25	("datetime", icu::datetime::provider::KEYS, REPO_VERSION),
26	("decimal", icu::decimal::provider::KEYS, REPO_VERSION),
27	("list", icu::list::provider::KEYS, REPO_VERSION),
28	("locale", icu::locale::provider::KEYS, REPO_VERSION),
29	("normalizer", icu::normalizer::provider::KEYS, REPO_VERSION),
30	("plurals", icu::plurals::provider::KEYS, REPO_VERSION),
31	("properties", icu::properties::provider::KEYS, REPO_VERSION),
32	("segmenter", icu::segmenter::provider::KEYS, REPO_VERSION),
33	("timezone", icu::timezone::provider::KEYS, REPO_VERSION),
34	(
35	"experimental",
36	icu::experimental::provider::KEYS,
37	EXPERIMENTAL_VERSION,
38	),
39	];
40
UNCOV 41	fn main() {	×
UNCOV 42	simple_logger::SimpleLogger::new()	×
43	.env()
44	.with_level(log::LevelFilter::Info)	×
45	.init()
46	.unwrap();
47
UNCOV 48	let args = std::env::args().skip(1).collect::<Vec<_>>();	×
49
UNCOV 50	let components = if args.is_empty() {	×
UNCOV 51	COMPONENTS	×
52	.iter()
53	.map(\|(krate, keys, version)\| (krate.to_string(), keys, version))	×
54	.collect::<Vec<_>>()
55	} else {
UNCOV 56	let map = std::collections::HashMap::<&str, (&'static [DataKey], &'static str)>::from_iter(	×
57	COMPONENTS	×
58	.iter()
59	.map(\|(krate, keys, version)\| (krate, (keys, *version))),	×
60	);
61	args.into_iter()	×
UNCOV 62	.filter_map(\|krate\| {	×
63	map.get(krate.as_str())	×
UNCOV 64	.map(\|(keys, version)\| (krate, keys, version))	×
65	})	×
66	.collect()
67	};	×
68
69	let source = DatagenProvider::new_latest_tested();	×
70
71	let driver = DatagenDriver::new()	×
72	.with_locales_and_fallback(
73	source	×
UNCOV 74	.locales_for_coverage_levels([	×
UNCOV 75	CoverageLevel::Modern,	×
UNCOV 76	CoverageLevel::Moderate,	×
77	CoverageLevel::Basic,	×
78	])
79	.unwrap()
80	.into_iter()
UNCOV 81	.map(LocaleFamily::with_descendants),	×
82	Default::default(),	×
83	)	×
84	.with_recommended_segmenter_models();
85
86	let mut options = baked_exporter::Options::default();	×
87	options.overwrite = true;	×
UNCOV 88	options.pretty = true;	×
89
90	for (component, keys, version) in &components {	×
91	let path = Path::new("provider/baked").join(component);	×
92
93	let _ = std::fs::remove_dir_all(&path);	×
94	for dir in ["", "src", "data"] {	×
95	std::fs::create_dir(&path.join(dir)).unwrap();	×
96	}
UNCOV 97	for (file, template) in [	×
UNCOV 98	("build.rs", include_str!("../template/build.rs.template")),	×
99	(	×
100	"Cargo.toml",
101	include_str!("../template/Cargo.toml.template"),
102	),
UNCOV 103	("LICENSE", include_str!("../LICENSE")),	×
UNCOV 104	("README.md", include_str!("../template/README.md.template")),	×
UNCOV 105	(	×
106	"src/lib.rs",
107	include_str!("../template/src/lib.rs.template"),
108	),
109	] {
110	std::fs::write(	×
UNCOV 111	path.join(file),	×
UNCOV 112	template	×
UNCOV 113	.replace("_component_", component)	×
UNCOV 114	.replace("_version_", version)	×
115	.replace("_cldr_tag_", DatagenProvider::LATEST_TESTED_CLDR_TAG)
116	.replace(
117	"_icuexport_tag_",
118	DatagenProvider::LATEST_TESTED_ICUEXPORT_TAG,
119	)
120	.replace(
121	"_segmenter_lstm_tag_",
122	DatagenProvider::LATEST_TESTED_SEGMENTER_LSTM_TAG,
123	),
124	)	×
125	.unwrap();	×
126	}
127
128	let baked_exporter =
129	baked_exporter::BakedExporter::new(path.join("data"), options).unwrap();	×
130	let fingerprinter = PostcardFingerprintExporter {	×
UNCOV 131	size_hash: Default::default(),	×
UNCOV 132	fingerprints: crlify::BufWriterWithLineEndingFix::new(	×
133	File::create(path.join("fingerprints.csv")).unwrap(),	×
134	),
UNCOV 135	};	×
136
137	driver	×
UNCOV 138	.clone()	×
UNCOV 139	.with_keys(keys.iter().copied())	×
140	.export(
141	&source,
UNCOV 142	MultiExporter::new(vec![Box::new(baked_exporter), Box::new(fingerprinter)]),	×
UNCOV 143	)	×
144	.unwrap();
145
UNCOV 146	for file in ["data/any.rs", "data/mod.rs"] {	×
UNCOV 147	std::fs::remove_file(path.join(file)).unwrap();	×
148	}
UNCOV 149	}	×
UNCOV 150	}	×
151
152	struct PostcardFingerprintExporter<F> {
153	size_hash: Mutex<BTreeMap<(DataKey, String), (usize, u64)>>,
154	fingerprints: F,
155	}
156
157	impl<F: Write + Send + Sync> DataExporter for PostcardFingerprintExporter<F> {
UNCOV 158	fn put_payload(	×
159	&self,
160	key: DataKey,
161	locale: &DataLocale,
162	key_attributes: &DataKeyAttributes,
163	payload_before: &DataPayload<ExportMarker>,
164	) -> Result<(), DataError> {
UNCOV 165	let mut serialized = vec![];	×
166
UNCOV 167	fs_exporter::serializers::Postcard::new(Default::default())	×
UNCOV 168	.serialize(payload_before, &mut serialized)?;	×
169
UNCOV 170	let size = serialized.len();	×
171
172	// We're using SipHash, which is deprecated, but we want a stable hasher
173	// (we're fine with it not being cryptographically secure since we're just using it to track diffs)
174	#[allow(deprecated)]
175	use std::hash::{Hash, Hasher, SipHasher};
176	#[allow(deprecated)]
UNCOV 177	let mut hasher = SipHasher::new();	×
UNCOV 178	serialized.iter().for_each(\|b\| b.hash(&mut hasher));	×
UNCOV 179	let hash = hasher.finish();	×
180
UNCOV 181	self.size_hash.lock().expect("poison").insert(	×
UNCOV 182	(	×
183	key,
UNCOV 184	DataRequest {	×
185	locale,
186	key_attributes,
UNCOV 187	..Default::default()	×
188	}
189	.legacy_encode(),
190	),
UNCOV 191	(size, hash),	×
UNCOV 192	);	×
193
UNCOV 194	Ok(())	×
UNCOV 195	}	×
196
UNCOV 197	fn flush(&self, _key: DataKey) -> Result<(), DataError> {	×
UNCOV 198	Ok(())	×
UNCOV 199	}	×
200
UNCOV 201	fn flush_with_built_in_fallback(	×
202	&self,
203	_key: DataKey,
UNCOV 204	_fallback_mode: BuiltInFallbackMode,	×
205	) -> Result<(), DataError> {
UNCOV 206	Ok(())	×
UNCOV 207	}	×
208
UNCOV 209	fn close(&mut self) -> Result<(), DataError> {	×
UNCOV 210	for ((key, req), (size, hash)) in self.size_hash.get_mut().expect("poison") {	×
UNCOV 211	writeln!(&mut self.fingerprints, "{key}, {req}, {size}B, {hash:x}")?;	×
212	}
UNCOV 213	Ok(())	×
UNCOV 214	}	×
UNCOV 215	fn supports_built_in_fallback(&self) -> bool {	×
216	true
UNCOV 217	}	×
218	}

zbraniecki / icu4x / 9357137046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous