13958601093

Committed 19 Mar 2025 04:17PM UTC coverage: 74.164% (-1.5%) from 75.71%

Build # 13958601093

Build Type

push

github

Committed by

web-flow

Commit Message

Clean up properties docs (#6315)

Run Details

58056 of 78281 relevant lines covered (74.16%)

819371.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.49

/provider/export/src/export_impl.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use crate::{DataLocaleFamilyAnnotations, DeduplicationStrategy, ExportDriver, ExportMetadata};
use icu_locale::fallback::LocaleFallbackIterator;
use icu_locale::LocaleFallbacker;
use icu_provider::export::*;
use icu_provider::prelude::*;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use std::time::Instant;
use writeable::Writeable;

#[cfg(not(feature = "rayon"))]
trait IntoParallelIterator: IntoIterator + Sized {
    fn into_par_iter(self) -> <Self as IntoIterator>::IntoIter {
        self.into_iter()
    }
}
#[cfg(not(feature = "rayon"))]
impl<T: IntoIterator> IntoParallelIterator for T {}
#[cfg(feature = "rayon")]
use rayon::prelude::*;

impl ExportDriver {
    pub(crate) fn export_dyn(
        self,
        provider: &dyn ExportableProvider,
        sink: &mut dyn DataExporter,
    ) -> Result<ExportMetadata, DataError> {
        let Self {
            markers,
            requested_families,
            include_full,
            fallbacker,
            deduplication_strategy,
            attributes_filters,
        } = self;

        let markers = markers.unwrap_or_else(|| provider.supported_markers());

        if markers.is_empty() {
            log::warn!("No markers selected");
        }

        log::info!(
            "Datagen configured with {}, and these locales: {:?}",
            match deduplication_strategy {
                DeduplicationStrategy::Maximal => "maximal deduplication",
                DeduplicationStrategy::RetainBaseLanguages =>
                    "deduplication retaining base languages",
                DeduplicationStrategy::None => "no deduplication",
            },
            if include_full {
                vec!["<all>".to_string()]
            } else {
                let mut sorted_locale_strs = requested_families
                    .iter()
                    .map(|(l, a)| {
                        let mut s = String::new();
                        let _infallible = a.write_to(&mut s);
                        let _infallible = l.write_to(&mut s);
                        s
                    })
                    .collect::<Vec<_>>();
                sorted_locale_strs.sort_unstable();
                sorted_locale_strs
            }
        );

        let load_with_fallback = |marker, id: DataIdentifierBorrowed<'_>| {
            log::trace!("Generating marker/locale: {marker:?}/{id}");
            let mut metadata = DataRequestMetadata::default();
            metadata.silent = true;
            // Lazy-compute the fallback iterator so that we don't always require CLDR data
            let mut locale_iter: Option<LocaleFallbackIterator> = None;
            loop {
                let req = DataRequest {
                    id: DataIdentifierBorrowed::for_marker_attributes_and_locale(
                        id.marker_attributes,
                        locale_iter.as_ref().map(|i| i.get()).unwrap_or(id.locale),
                    ),
                    metadata,
                };
                match provider.load_data(marker, req).allow_identifier_not_found() {
                    Ok(Some(data_response)) => {
                        if let Some(iter) = locale_iter.as_ref() {
                            if iter.get().is_default() && !id.locale.is_default() {
                                log::debug!("Falling back to und: {marker:?}/{id}");
                            }
                        }
                        return Some(Ok(data_response));
                    }
                    Ok(None) => {
                        if let Some(iter) = locale_iter.as_mut() {
                            if iter.get().is_default() {
                                log::debug!("Could not find data for: {marker:?}/{id}");
                                return None;
                            }
                            iter.step();
                        } else {
                            locale_iter = Some(
                                fallbacker
                                    .for_config(marker.fallback_config)
                                    .fallback_for(*id.locale),
                            )
                        }
                    }
                    Err(e) => return Some(Err(e.with_req(marker, req))),
                }
            }
        };

        markers.clone().into_par_iter().try_for_each(|marker| {
            log::trace!("Generating marker {marker:?}");
            let instant1 = Instant::now();

            let mut flush_metadata = FlushMetadata::default();
            flush_metadata.supports_dry_provider = matches!(
                deduplication_strategy,
                DeduplicationStrategy::RetainBaseLanguages | DeduplicationStrategy::None
            );

            if marker.is_singleton {
                let supported = provider.iter_ids_for_marker(marker)?;
                if supported.len() != 1 || !supported.first().unwrap().is_default() {
                    return Err(DataError::custom(
                        "Invalid supported locales for singleton marker",
                    )
                    .with_marker(marker));
                }

                let response = provider
                    .load_data(marker, Default::default())
                    .map_err(|e| e.with_req(marker, Default::default()))?;

                let transform_duration = instant1.elapsed();

                if marker.has_checksum {
                    flush_metadata.checksum = response.metadata.checksum;
                } else if response.metadata.checksum.is_some() {
                    log::warn!("{marker:?} returns a checksum, but it's not configured to");
                }

                sink.flush_singleton(marker, &response.payload, flush_metadata)
                    .map_err(|e| e.with_req(marker, Default::default()))?;

                let final_duration = instant1.elapsed();
                let flush_duration = final_duration - transform_duration;

                if final_duration > Duration::new(0, 500_000_000) {
                    // Print durations if the marker took longer than 500 ms
                    log::info!(
                        "Generated marker {marker:?} ({}, flushed in {})",
                        DisplayDuration(final_duration),
                        DisplayDuration(flush_duration)
                    );
                } else {
                    log::info!("Generated marker {marker:?}");
                }
                return Ok(());
            }

            let locales_to_export = select_locales_for_marker(
                provider,
                marker,
                &requested_families,
                &attributes_filters,
                include_full,
                &fallbacker,
            )?;

            let responses = locales_to_export
                .into_par_iter()
                .filter_map(|id| {
                    let instant2 = Instant::now();
                    load_with_fallback(marker, id.as_borrowed())
                        .map(|r| r.map(move |payload| (id, (payload, instant2.elapsed()))))
                })
                .collect::<Result<HashMap<_, _>, _>>()?;

            if marker.has_checksum {
                flush_metadata.checksum =
                    responses
                        .iter()
                        .try_fold(None, |acc, (id, (response, _))| {
                            match (acc, response.metadata.checksum) {
                                (Some(a), Some(b)) if a != b => {
                                    Err(DataError::custom("Mismatched checksums").with_req(
                                        marker,
                                        DataRequest {
                                            id: id.as_borrowed(),
                                            ..Default::default()
                                        },
                                    ))
                                }
                                (a, b) => Ok(a.or(b)),
                            }
                        })?;
            } else if responses.iter().any(|r| r.1 .0.metadata.checksum.is_some()) {
                log::warn!("{marker:?} returns a checksum, but it's not configured to");
            }

            let (slowest_duration, slowest_id) = match deduplication_strategy {
                DeduplicationStrategy::Maximal | DeduplicationStrategy::RetainBaseLanguages => {
                    deduplicate_responses(
                        deduplication_strategy == DeduplicationStrategy::Maximal,
                        marker,
                        responses,
                        &fallbacker,
                        sink,
                    )?
                }
                DeduplicationStrategy::None => responses
                    .into_iter()
                    .map(|(id, (response, time))| {
                        sink.put_payload(marker, id.as_borrowed(), &response.payload)
                            .map_err(|e| {
                                e.with_req(
                                    marker,
                                    DataRequest {
                                        id: id.as_borrowed(),
                                        ..Default::default()
                                    },
                                )
                            })
                            .map(|()| (time, id))
                    })
                    .collect::<Result<Vec<_>, DataError>>()?
                    .into_iter()
                    .max(),
            }
            .unwrap_or_default();

            let transform_duration = instant1.elapsed();

            sink.flush(marker, flush_metadata)
                .map_err(|e| e.with_marker(marker))?;

            let final_duration = instant1.elapsed();
            let flush_duration = final_duration - transform_duration;

            if final_duration > Duration::new(0, 500_000_000) {
                // Print durations if the marker took longer than 500 ms
                log::info!(
                    "Generated marker {marker:?} ({}, '{}/{}' in {}, flushed in {})",
                    DisplayDuration(final_duration),
                    slowest_id.locale,
                    slowest_id.marker_attributes.as_str(),
                    DisplayDuration(slowest_duration),
                    DisplayDuration(flush_duration)
                );
            } else {
                log::info!("Generated marker {marker:?}");
            }
            Ok(())
        })?;

        let exporter = sink.close()?;

        Ok(ExportMetadata { exporter })
    }
}

/// Selects the maximal set of locales to export based on a [`DataMarkerInfo`] and this datagen
/// provider's options bag. The locales may be later optionally deduplicated for fallback.
#[allow(clippy::type_complexity)] // sigh
fn select_locales_for_marker<'a>(
    provider: &'a dyn ExportableProvider,
    marker: DataMarkerInfo,
    requested_families: &HashMap<DataLocale, DataLocaleFamilyAnnotations>,
    attributes_filters: &HashMap<
        String,
        Arc<Box<dyn Fn(&DataMarkerAttributes) -> bool + Send + Sync + 'static>>,
    >,
    include_full: bool,
    fallbacker: &LocaleFallbacker,
) -> Result<HashSet<DataIdentifierCow<'a>>, DataError> {
    // Map from all supported DataLocales to their corresponding supported DataIdentifiers.
    let mut supported_map = HashMap::<DataLocale, HashSet<DataIdentifierCow<'a>>>::new();
    for id in provider
        .iter_ids_for_marker(marker)
        .map_err(|e| e.with_marker(marker))?
    {
        supported_map.entry(id.locale).or_default().insert(id);
    }

    if !marker.attributes_domain.is_empty() {
        if let Some(filter) = attributes_filters.get(marker.attributes_domain) {
            supported_map.retain(|_, ids| {
                ids.retain(|id| filter(&id.marker_attributes));
                !ids.is_empty()
            });
        }
    }

    if include_full && requested_families.is_empty() {
        // Special case: return now so we don't need the fallbacker (and its requisite CLDR data)
        let selected_locales = supported_map.into_values().flatten().collect();
        return Ok(selected_locales);
    }

    // The "candidate" locales that could be exported is the union of requested and supported.
    let all_candidate_locales = supported_map
        .keys()
        .chain(requested_families.keys())
        .collect::<HashSet<_>>();

    // Compute a map from LanguageIdentifiers to DataLocales, including inherited auxiliary keys
    // and extensions. Also resolve the ancestors and descendants while building this map.
    let mut selected_locales = requested_families.keys().cloned().collect::<HashSet<_>>();
    let expansion_map: HashMap<&DataLocale, HashSet<DataIdentifierCow>> = all_candidate_locales
        .into_iter()
        .map(|current_locale| {
            let mut expansion = supported_map
                .get(current_locale)
                .cloned()
                .unwrap_or_default();
            if include_full && !selected_locales.contains(current_locale) {
                log::trace!("Including {current_locale}: full locale family: {marker:?}");
                selected_locales.insert(*current_locale);
            }
            if current_locale.language.is_default() && !current_locale.is_default() {
                log::trace!("Including {current_locale}: und variant: {marker:?}");
                selected_locales.insert(*current_locale);
            }
            let include_ancestors = requested_families
                .get(current_locale)
                .map(|family| family.include_ancestors)
                // default to `false` if the locale was not requested
                .unwrap_or(false);
            let mut iter = fallbacker
                .for_config(marker.fallback_config)
                .fallback_for(*current_locale);
            loop {
                // Inherit aux keys and extension keywords from parent locales
                let parent_locale = iter.get();
                let maybe_parent_ids = supported_map.get(parent_locale);
                let include_descendants = requested_families
                    .get(parent_locale)
                    .map(|family| family.include_descendants)
                    // default to `false` if the locale was not requested
                    .unwrap_or(false);
                if include_descendants && !selected_locales.contains(current_locale) {
                    log::trace!(
                        "Including {current_locale}: descendant of {parent_locale}: {marker:?}"
                    );
                    selected_locales.insert(*current_locale);
                }
                if include_ancestors && !selected_locales.contains(parent_locale) {
                    log::trace!(
                        "Including {parent_locale}: ancestor of {current_locale}: {marker:?}"
                    );
                    selected_locales.insert(*parent_locale);
                }
                if let Some(parent_ids) = maybe_parent_ids {
                    for morphed_id in parent_ids.iter() {
                        // Special case: don't pull extensions or aux keys up from the root.
                        if morphed_id.locale.is_default() && !morphed_id.is_default() {
                            continue;
                        }
                        let mut morphed_id = morphed_id.clone();
                        morphed_id.locale = *current_locale;
                        expansion.insert(morphed_id);
                    }
                }
                if iter.get().is_default() {
                    break;
                }
                iter.step();
            }
            (current_locale, expansion)
        })
        .collect();

    let selected_locales = expansion_map
        .into_iter()
        .filter(|(locale, _)| selected_locales.contains(locale))
        .flat_map(|(_, data_locales)| data_locales)
        .collect();
    Ok(selected_locales)
}

fn deduplicate_responses<'a>(
    maximal: bool,
    marker: DataMarkerInfo,
    responses: HashMap<DataIdentifierCow<'a>, (DataResponse<ExportMarker>, Duration)>,
    fallbacker: &LocaleFallbacker,
    sink: &dyn DataExporter,
) -> Result<Option<(Duration, DataIdentifierCow<'a>)>, DataError> {
    let fallbacker_with_config = fallbacker.for_config(marker.fallback_config);
    responses
        .iter()
        .try_for_each(|(id, (response, _duration))| {
            // Always export `und`. This prevents calling `step` on an empty locale.
            if id.locale.is_default() {
                return sink
                    .put_payload(marker, id.as_borrowed(), &response.payload)
                    .map_err(|e| {
                        e.with_req(
                            marker,
                            DataRequest {
                                id: id.as_borrowed(),
                                ..Default::default()
                            },
                        )
                    });
            }
            let mut iter = fallbacker_with_config.fallback_for(id.locale);
            loop {
                if !maximal {
                    // To retain base languages, preemptively step to the
                    // parent locale. This should retain the locale if
                    // the next parent is `und`.
                    iter.step();
                }
                if iter.get().is_default() {
                    break;
                }
                if maximal {
                    iter.step();
                }

                if let Some((inherited_response, _duration)) = responses.get(
                    &DataIdentifierBorrowed::for_marker_attributes_and_locale(
                        &id.marker_attributes,
                        iter.get(),
                    )
                    .as_cow(),
                ) {
                    if inherited_response.payload == response.payload {
                        // Found a match: don't need to write anything
                        log::trace!("Deduplicating {id} (inherits from {})", iter.get());
                        return Ok(());
                    } else {
                        // Not a match: we must include this
                        break;
                    }
                }
            }
            // Did not find a match: export this payload
            sink.put_payload(marker, id.as_borrowed(), &response.payload)
                .map_err(|e| {
                    e.with_req(
                        marker,
                        DataRequest {
                            id: id.as_borrowed(),
                            ..Default::default()
                        },
                    )
                })
        })?;

    // Slowest locale calculation:
    Ok(responses
        .into_iter()
        .map(|(id, (_response, duration))| (duration, id))
        .max())
}

struct DisplayDuration(pub Duration);

impl fmt::Display for DisplayDuration {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let nanos = self.0.as_nanos();
        if nanos > 100_000_000 {
            write!(f, "{:.3}s", self.0.as_secs_f64())
        } else if nanos > 1_000_000 {
            write!(f, "{:.3}ms", (nanos as f64) / 1e6)
        } else if nanos > 1_000 {
            write!(f, "{:.3}µs", (nanos as f64) / 1e3)
        } else {
            write!(f, "{}ns", nanos)
        }
    }
}

#[test]
fn test_collation_filtering() {
    use crate::DataLocaleFamily;
    use icu::locale::locale;
    use std::collections::BTreeSet;

    struct Provider;

    impl DataProvider<icu::collator::provider::CollationTailoringV1> for Provider {
        fn load(
            &self,
            _req: DataRequest,
        ) -> Result<DataResponse<icu::collator::provider::CollationTailoringV1>, DataError>
        {
            unreachable!()
        }
    }

    impl IterableDataProvider<icu::collator::provider::CollationTailoringV1> for Provider {
        fn iter_ids(&self) -> Result<BTreeSet<DataIdentifierCow>, DataError> {
            Ok(BTreeSet::from_iter(
                [
                    (locale!("ko"), "search"),
                    (locale!("ko"), "searchjl"),
                    (locale!("ko"), "unihan"),
                    (locale!("ko"), ""),
                    (locale!("und"), "emoji"),
                    (locale!("und"), "eor"),
                    (locale!("und"), "search"),
                    (locale!("und"), ""),
                    (locale!("zh"), "stroke"),
                    (locale!("zh"), "unihan"),
                    (locale!("zh"), "zhuyin"),
                    (locale!("zh"), ""),
                ]
                .into_iter()
                .map(|(l, a)| {
                    DataIdentifierCow::from_borrowed_and_owned(
                        DataMarkerAttributes::from_str_or_panic(a),
                        l.into(),
                    )
                }),
            ))
        }
    }

    extern crate alloc;
    icu_provider::export::make_exportable_provider!(
        Provider,
        [icu::collator::provider::CollationTailoringV1,]
    );

    #[derive(Debug)]
    struct TestCase<'a> {
        include_collations: &'a [&'a str],
        language: DataLocale,
        expected: &'a [&'a str],
    }
    let cases = [
        TestCase {
            include_collations: &[],
            language: locale!("zh").into(),
            expected: &["", "stroke", "unihan", "zhuyin"],
        },
        TestCase {
            include_collations: &["search*"],
            language: locale!("zh").into(),
            expected: &["", "stroke", "unihan", "zhuyin"],
        },
        TestCase {
            include_collations: &[],
            language: locale!("ko").into(),
            expected: &["", "unihan"],
        },
        TestCase {
            include_collations: &["search"],
            language: locale!("ko").into(),
            expected: &["", "search", "unihan"],
        },
        TestCase {
            include_collations: &["searchjl"],
            language: locale!("ko").into(),
            expected: &["", "searchjl", "unihan"],
        },
        TestCase {
            include_collations: &["search", "searchjl"],
            language: locale!("ko").into(),
            expected: &["", "search", "searchjl", "unihan"],
        },
        TestCase {
            include_collations: &["search*"],
            language: locale!("ko").into(),
            expected: &["", "search", "searchjl", "unihan"],
        },
        TestCase {
            include_collations: &[],
            language: locale!("und").into(),
            expected: &["", "emoji", "eor"],
        },
    ];
    for cas in cases {
        let driver = ExportDriver::new(
            [DataLocaleFamily::single(cas.language)],
            DeduplicationStrategy::None.into(),
            LocaleFallbacker::new_without_data(),
        )
        .with_additional_collations(cas.include_collations.iter().copied().map(String::from));
        let resolved_locales = select_locales_for_marker(
            &Provider,
            icu::collator::provider::CollationTailoringV1::INFO,
            &driver.requested_families,
            &driver.attributes_filters,
            false,
            &driver.fallbacker,
        )
        .unwrap()
        .into_iter()
        .map(|id| id.marker_attributes.to_string())
        .collect::<BTreeSet<_>>();
        let expected_locales = cas
            .expected
            .iter()
            .copied()
            .map(String::from)
            .collect::<BTreeSet<_>>();
        assert_eq!(resolved_locales, expected_locales, "{cas:?}");
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	use crate::{DataLocaleFamilyAnnotations, DeduplicationStrategy, ExportDriver, ExportMetadata};
6	use icu_locale::fallback::LocaleFallbackIterator;
7	use icu_locale::LocaleFallbacker;
8	use icu_provider::export::*;
9	use icu_provider::prelude::*;
10	use std::collections::HashMap;
11	use std::collections::HashSet;
12	use std::fmt;
13	use std::sync::Arc;
14	use std::time::Duration;
15	use std::time::Instant;
16	use writeable::Writeable;
17
18	#[cfg(not(feature = "rayon"))]
19	trait IntoParallelIterator: IntoIterator + Sized {
20	fn into_par_iter(self) -> <Self as IntoIterator>::IntoIter {
21	self.into_iter()
22	}
23	}
24	#[cfg(not(feature = "rayon"))]
25	impl<T: IntoIterator> IntoParallelIterator for T {}
26	#[cfg(feature = "rayon")]
27	use rayon::prelude::*;
28
29	impl ExportDriver {
30	pub(crate) fn export_dyn(	63✔
31	self,
32	provider: &dyn ExportableProvider,
33	sink: &mut dyn DataExporter,
34	) -> Result<ExportMetadata, DataError> {
35	let Self {
36	markers,	63✔
37	requested_families,	63✔
38	include_full,	63✔
39	fallbacker,	63✔
40	deduplication_strategy,	63✔
41	attributes_filters,	63✔
42	} = self;
43
44	let markers = markers.unwrap_or_else(\|\| provider.supported_markers());	87✔
45
46	if markers.is_empty() {	63✔
47	log::warn!("No markers selected");	×
48	}
49
50	log::info!(	24✔
51	"Datagen configured with {}, and these locales: {:?}",
52	match deduplication_strategy {
53	DeduplicationStrategy::Maximal => "maximal deduplication",
54	DeduplicationStrategy::RetainBaseLanguages =>
55	"deduplication retaining base languages",
56	DeduplicationStrategy::None => "no deduplication",
57	},
58	if include_full {
59	vec!["<all>".to_string()]
60	} else {
61	let mut sorted_locale_strs = requested_families
62	.iter()
63	.map(\|(l, a)\| {	58✔
64	let mut s = String::new();	58✔
65	let _infallible = a.write_to(&mut s);	58✔
66	let _infallible = l.write_to(&mut s);	58✔
67	s	58✔
68	})	58✔
69	.collect::<Vec<_>>();
70	sorted_locale_strs.sort_unstable();	16✔
71	sorted_locale_strs	16✔
72	}
73	);
74
75	let load_with_fallback = \|marker, id: DataIdentifierBorrowed<'_>\| {	7,810✔
76	log::trace!("Generating marker/locale: {marker:?}/{id}");	7,767✔
77	let mut metadata = DataRequestMetadata::default();	7,767✔
78	metadata.silent = true;	7,767✔
79	// Lazy-compute the fallback iterator so that we don't always require CLDR data
80	let mut locale_iter: Option<LocaleFallbackIterator> = None;	7,767✔
81	loop {	8,050✔
82	let req = DataRequest {	8,050✔
83	id: DataIdentifierBorrowed::for_marker_attributes_and_locale(	8,050✔
84	id.marker_attributes,	8,050✔
85	locale_iter.as_ref().map(\|i\| i.get()).unwrap_or(id.locale),	8,334✔
86	),
87	metadata,	8,050✔
88	};
89	match provider.load_data(marker, req).allow_identifier_not_found() {	8,050✔
90	Ok(Some(data_response)) => {	7,767✔
91	if let Some(iter) = locale_iter.as_ref() {	7,767✔
92	if iter.get().is_default() && !id.locale.is_default() {	122✔
93	log::debug!("Falling back to und: {marker:?}/{id}");	38✔
94	}
95	}
96	return Some(Ok(data_response));	7,767✔
97	}	×
98	Ok(None) => {
99	if let Some(iter) = locale_iter.as_mut() {	283✔
100	if iter.get().is_default() {	162✔
101	log::debug!("Could not find data for: {marker:?}/{id}");	×
102	return None;	×
103	}
104	iter.step();	162✔
105	} else {
106	locale_iter = Some(	121✔
107	fallbacker	363✔
108	.for_config(marker.fallback_config)	121✔
109	.fallback_for(*id.locale),	121✔
110	)
111	}
112	}
113	Err(e) => return Some(Err(e.with_req(marker, req))),	×
114	}
115	}	8,050✔
116	};	7,767✔
117
118	markers.clone().into_par_iter().try_for_each(\|marker\| {	367✔
119	log::trace!("Generating marker {marker:?}");	299✔
120	let instant1 = Instant::now();	299✔
121
122	let mut flush_metadata = FlushMetadata::default();	299✔
123	flush_metadata.supports_dry_provider = matches!(	299✔
124	deduplication_strategy,	299✔
125	DeduplicationStrategy::RetainBaseLanguages \| DeduplicationStrategy::None
126	);
127
128	if marker.is_singleton {	299✔
129	let supported = provider.iter_ids_for_marker(marker)?;	156✔
130	if supported.len() != 1 \|\| !supported.first().unwrap().is_default() {	156✔
131	return Err(DataError::custom(	×
132	"Invalid supported locales for singleton marker",
133	)
134	.with_marker(marker));
135	}
136
137	let response = provider	156✔
138	.load_data(marker, Default::default())	156✔
139	.map_err(\|e\| e.with_req(marker, Default::default()))?;	×
140
141	let transform_duration = instant1.elapsed();	156✔
142
143	if marker.has_checksum {	156✔
144	flush_metadata.checksum = response.metadata.checksum;	3✔
145	} else if response.metadata.checksum.is_some() {	153✔
146	log::warn!("{marker:?} returns a checksum, but it's not configured to");	×
147	}
148
149	sink.flush_singleton(marker, &response.payload, flush_metadata)	156✔
150	.map_err(\|e\| e.with_req(marker, Default::default()))?;	×
151
152	let final_duration = instant1.elapsed();	156✔
153	let flush_duration = final_duration - transform_duration;	156✔
154
155	if final_duration > Duration::new(0, 500_000_000) {	156✔
156	// Print durations if the marker took longer than 500 ms
157	log::info!(	15✔
158	"Generated marker {marker:?} ({}, flushed in {})",
159	DisplayDuration(final_duration),
160	DisplayDuration(flush_duration)
161	);
162	} else {
163	log::info!("Generated marker {marker:?}");	141✔
164	}
165	return Ok(());	156✔
166	}	156✔
167
168	let locales_to_export = select_locales_for_marker(	143✔
169	provider,	143✔
170	marker,
171	&requested_families,	143✔
172	&attributes_filters,	143✔
173	include_full,	143✔
174	&fallbacker,	143✔
175	)?;	×
176
177	let responses = locales_to_export	143✔
178	.into_par_iter()
179	.filter_map(\|id\| {	7,912✔
180	let instant2 = Instant::now();	7,769✔
181	load_with_fallback(marker, id.as_borrowed())	7,773✔
182	.map(\|r\| r.map(move \|payload\| (id, (payload, instant2.elapsed()))))	23,290✔
183	})	7,741✔
184	.collect::<Result<HashMap<_, _>, _>>()?;	×
185
186	if marker.has_checksum {	171✔
187	flush_metadata.checksum =	28✔
188	responses	28✔
189	.iter()
190	.try_fold(None, \|acc, (id, (response, _))\| {	404✔
191	match (acc, response.metadata.checksum) {	376✔
192	(Some(a), Some(b)) if a != b => {	163✔
193	Err(DataError::custom("Mismatched checksums").with_req(	×
194	marker,	×
195	DataRequest {	×
196	id: id.as_borrowed(),	×
197	..Default::default()	×
198	},
199	))
200	}	×
201	(a, b) => Ok(a.or(b)),	376✔
202	}
203	})?;	376✔
204	} else if responses.iter().any(\|r\| r.1 .0.metadata.checksum.is_some()) {	7,510✔
205	log::warn!("{marker:?} returns a checksum, but it's not configured to");	×
206	}
207
208	let (slowest_duration, slowest_id) = match deduplication_strategy {	286✔
209	DeduplicationStrategy::Maximal \| DeduplicationStrategy::RetainBaseLanguages => {
210	deduplicate_responses(	11✔
211	deduplication_strategy == DeduplicationStrategy::Maximal,	11✔
212	marker,
213	responses,	11✔
214	&fallbacker,	11✔
215	sink,	11✔
216	)?	×
217	}	11✔
218	DeduplicationStrategy::None => responses	132✔
219	.into_iter()
220	.map(\|(id, (response, time))\| {	7,741✔
221	sink.put_payload(marker, id.as_borrowed(), &response.payload)	7,609✔
222	.map_err(\|e\| {	7,609✔
223	e.with_req(	×
224	marker,	×
225	DataRequest {	×
226	id: id.as_borrowed(),	×
227	..Default::default()	×
228	},
229	)
230	})	×
231	.map(\|()\| (time, id))	15,216✔
232	})	7,607✔
233	.collect::<Result<Vec<_>, DataError>>()?	×
234	.into_iter()
235	.max(),
236	}
237	.unwrap_or_default();
238
239	let transform_duration = instant1.elapsed();	143✔
240
241	sink.flush(marker, flush_metadata)	442✔
242	.map_err(\|e\| e.with_marker(marker))?;	×
243
244	let final_duration = instant1.elapsed();	143✔
245	let flush_duration = final_duration - transform_duration;	143✔
246
247	if final_duration > Duration::new(0, 500_000_000) {	143✔
248	// Print durations if the marker took longer than 500 ms
249	log::info!(	21✔
250	"Generated marker {marker:?} ({}, '{}/{}' in {}, flushed in {})",
251	DisplayDuration(final_duration),
252	slowest_id.locale,
253	slowest_id.marker_attributes.as_str(),
254	DisplayDuration(slowest_duration),
255	DisplayDuration(flush_duration)
256	);
257	} else {
258	log::info!("Generated marker {marker:?}");	122✔
259	}
260	Ok(())	143✔
261	})?;	299✔
262
263	let exporter = sink.close()?;	25✔
264
265	Ok(ExportMetadata { exporter })	25✔
266	}	25✔
267	}
268
269	/// Selects the maximal set of locales to export based on a [`DataMarkerInfo`] and this datagen
270	/// provider's options bag. The locales may be later optionally deduplicated for fallback.
271	#[allow(clippy::type_complexity)] // sigh
272	fn select_locales_for_marker<'a>(	163✔
273	provider: &'a dyn ExportableProvider,
274	marker: DataMarkerInfo,
275	requested_families: &HashMap<DataLocale, DataLocaleFamilyAnnotations>,
276	attributes_filters: &HashMap<
277	String,
278	Arc<Box<dyn Fn(&DataMarkerAttributes) -> bool + Send + Sync + 'static>>,
279	>,
280	include_full: bool,
281	fallbacker: &LocaleFallbacker,
282	) -> Result<HashSet<DataIdentifierCow<'a>>, DataError> {
283	// Map from all supported DataLocales to their corresponding supported DataIdentifiers.
284	let mut supported_map = HashMap::<DataLocale, HashSet<DataIdentifierCow<'a>>>::new();	163✔
285	for id in provider	25,802✔
286	.iter_ids_for_marker(marker)
287	.map_err(\|e\| e.with_marker(marker))?	×
288	{
289	supported_map.entry(id.locale).or_default().insert(id);	25,651✔
290	}	25,790✔
291
292	if !marker.attributes_domain.is_empty() {	151✔
293	if let Some(filter) = attributes_filters.get(marker.attributes_domain) {	45✔
294	supported_map.retain(\|_, ids\| {	130✔
295	ids.retain(\|id\| filter(&id.marker_attributes));	17,827✔
296	!ids.is_empty()	112✔
297	});	112✔
298	}
299	}
300
301	if include_full && requested_families.is_empty() {	151✔
302	// Special case: return now so we don't need the fallbacker (and its requisite CLDR data)
303	let selected_locales = supported_map.into_values().flatten().collect();	9✔
304	return Ok(selected_locales);	9✔
305	}
306
307	// The "candidate" locales that could be exported is the union of requested and supported.
308	let all_candidate_locales = supported_map	284✔
309	.keys()
310	.chain(requested_families.keys())	142✔
311	.collect::<HashSet<_>>();	142✔
312
313	// Compute a map from LanguageIdentifiers to DataLocales, including inherited auxiliary keys
314	// and extensions. Also resolve the ancestors and descendants while building this map.
315	let mut selected_locales = requested_families.keys().cloned().collect::<HashSet<_>>();	142✔
316	let expansion_map: HashMap<&DataLocale, HashSet<DataIdentifierCow>> = all_candidate_locales	142✔
317	.into_iter()
318	.map(\|current_locale\| {	3,237✔
319	let mut expansion = supported_map	3,095✔
320	.get(current_locale)
321	.cloned()
322	.unwrap_or_default();
323	if include_full && !selected_locales.contains(current_locale) {	3,095✔
324	log::trace!("Including {current_locale}: full locale family: {marker:?}");	×
325	selected_locales.insert(*current_locale);	×
326	}
327	if current_locale.language.is_default() && !current_locale.is_default() {	3,095✔
328	log::trace!("Including {current_locale}: und variant: {marker:?}");	157✔
329	selected_locales.insert(*current_locale);	157✔
330	}
331	let include_ancestors = requested_families	3,094✔
332	.get(current_locale)
333	.map(\|family\| family.include_ancestors)	2,190✔
334	// default to `false` if the locale was not requested
335	.unwrap_or(false);
336	let mut iter = fallbacker	6,190✔
337	.for_config(marker.fallback_config)	3,095✔
338	.fallback_for(*current_locale);	3,095✔
339	loop {
340	// Inherit aux keys and extension keywords from parent locales
341	let parent_locale = iter.get();	6,971✔
342	let maybe_parent_ids = supported_map.get(parent_locale);	6,971✔
343	let include_descendants = requested_families	6,971✔
344	.get(parent_locale)
345	.map(\|family\| family.include_descendants)	5,466✔
346	// default to `false` if the locale was not requested
347	.unwrap_or(false);
348	if include_descendants && !selected_locales.contains(current_locale) {	6,971✔
349	log::trace!(	6✔
350	"Including {current_locale}: descendant of {parent_locale}: {marker:?}"
351	);
352	selected_locales.insert(*current_locale);	6✔
353	}
354	if include_ancestors && !selected_locales.contains(parent_locale) {	6,971✔
355	log::trace!(	197✔
356	"Including {parent_locale}: ancestor of {current_locale}: {marker:?}"
357	);
358	selected_locales.insert(*parent_locale);	197✔
359	}
360	if let Some(parent_ids) = maybe_parent_ids {	6,971✔
361	for morphed_id in parent_ids.iter() {	18,068✔
362	// Special case: don't pull extensions or aux keys up from the root.
363	if morphed_id.locale.is_default() && !morphed_id.is_default() {	18,148✔
364	continue;
365	}
366	let mut morphed_id = morphed_id.clone();	18,148✔
367	morphed_id.locale = *current_locale;	11,794✔
368	expansion.insert(morphed_id);	11,788✔
369	}
370	}
371	if iter.get().is_default() {	6,965✔
372	break;
373	}
374	iter.step();	3,875✔
375	}
376	(current_locale, expansion)	1,881✔
377	})	1,881✔
378	.collect();
379
380	let selected_locales = expansion_map	142✔
381	.into_iter()
382	.filter(\|(locale, _)\| selected_locales.contains(locale))	3,095✔
383	.flat_map(\|(_, data_locales)\| data_locales)	2,384✔
384	.collect();
385	Ok(selected_locales)	142✔
386	}	151✔
387
388	fn deduplicate_responses<'a>(	11✔
389	maximal: bool,
390	marker: DataMarkerInfo,
391	responses: HashMap<DataIdentifierCow<'a>, (DataResponse<ExportMarker>, Duration)>,
392	fallbacker: &LocaleFallbacker,
393	sink: &dyn DataExporter,
394	) -> Result<Option<(Duration, DataIdentifierCow<'a>)>, DataError> {
395	let fallbacker_with_config = fallbacker.for_config(marker.fallback_config);	11✔
396	responses	11✔
397	.iter()
398	.try_for_each(\|(id, (response, _duration))\| {	174✔
399	// Always export `und`. This prevents calling `step` on an empty locale.
400	if id.locale.is_default() {	163✔
401	return sink	18✔
402	.put_payload(marker, id.as_borrowed(), &response.payload)	6✔
403	.map_err(\|e\| {	6✔
404	e.with_req(	×
405	marker,	×
406	DataRequest {	×
407	id: id.as_borrowed(),	×
408	..Default::default()	×
409	},
410	)
411	});	×
412	}
413	let mut iter = fallbacker_with_config.fallback_for(id.locale);	157✔
414	loop {
415	if !maximal {	226✔
416	// To retain base languages, preemptively step to the
417	// parent locale. This should retain the locale if
418	// the next parent is `und`.
419	iter.step();	40✔
420	}
421	if iter.get().is_default() {	226✔
422	break;
423	}
424	if maximal {	137✔
425	iter.step();	121✔
426	}
427
428	if let Some((inherited_response, _duration)) = responses.get(	274✔
429	&DataIdentifierBorrowed::for_marker_attributes_and_locale(	137✔
430	&id.marker_attributes,	137✔
431	iter.get(),	137✔
432	)
433	.as_cow(),
434	) {
435	if inherited_response.payload == response.payload {	68✔
436	// Found a match: don't need to write anything
437	log::trace!("Deduplicating {id} (inherits from {})", iter.get());	30✔
438	return Ok(());	30✔
439	} else {
440	// Not a match: we must include this
441	break;
442	}
443	}
444	}	137✔
445	// Did not find a match: export this payload
446	sink.put_payload(marker, id.as_borrowed(), &response.payload)	254✔
447	.map_err(\|e\| {	127✔
448	e.with_req(	×
449	marker,	×
450	DataRequest {	×
451	id: id.as_borrowed(),	×
452	..Default::default()	×
453	},
454	)
455	})	×
456	})?;	163✔
457
458	// Slowest locale calculation:
459	Ok(responses	11✔
460	.into_iter()
461	.map(\|(id, (_response, duration))\| (duration, id))	163✔
462	.max())
463	}	11✔
464
465	struct DisplayDuration(pub Duration);
466
467	impl fmt::Display for DisplayDuration {
468	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	×
469	let nanos = self.0.as_nanos();	×
470	if nanos > 100_000_000 {	×
471	write!(f, "{:.3}s", self.0.as_secs_f64())	×
472	} else if nanos > 1_000_000 {	×
473	write!(f, "{:.3}ms", (nanos as f64) / 1e6)	×
474	} else if nanos > 1_000 {	×
475	write!(f, "{:.3}µs", (nanos as f64) / 1e3)	×
476	} else {
477	write!(f, "{}ns", nanos)	×
478	}
479	}	×
480	}
481
482	#[test]
483	fn test_collation_filtering() {	2✔
484	use crate::DataLocaleFamily;
485	use icu::locale::locale;
486	use std::collections::BTreeSet;
487
488	struct Provider;
489
490	impl DataProvider<icu::collator::provider::CollationTailoringV1> for Provider {
491	fn load(	×
492	&self,
493	_req: DataRequest,
494	) -> Result<DataResponse<icu::collator::provider::CollationTailoringV1>, DataError>
495	{
496	unreachable!()	×
497	}
498	}
499
500	impl IterableDataProvider<icu::collator::provider::CollationTailoringV1> for Provider {
501	fn iter_ids(&self) -> Result<BTreeSet<DataIdentifierCow>, DataError> {	8✔
502	Ok(BTreeSet::from_iter(	8✔
503	[	8✔
504	(locale!("ko"), "search"),	8✔
505	(locale!("ko"), "searchjl"),	8✔
506	(locale!("ko"), "unihan"),	8✔
507	(locale!("ko"), ""),	8✔
508	(locale!("und"), "emoji"),	8✔
509	(locale!("und"), "eor"),	8✔
510	(locale!("und"), "search"),	8✔
511	(locale!("und"), ""),	8✔
512	(locale!("zh"), "stroke"),	8✔
513	(locale!("zh"), "unihan"),	8✔
514	(locale!("zh"), "zhuyin"),	8✔
515	(locale!("zh"), ""),	8✔
516	]
517	.into_iter()
518	.map(\|(l, a)\| {	96✔
519	DataIdentifierCow::from_borrowed_and_owned(	96✔
520	DataMarkerAttributes::from_str_or_panic(a),	96✔
521	l.into(),	96✔
522	)
523	}),	96✔
524	))
525	}	8✔
526	}
527
528	extern crate alloc;
529	icu_provider::export::make_exportable_provider!(
530	Provider,
531	[icu::collator::provider::CollationTailoringV1,]
532	);
533
534	#[derive(Debug)]	×
535	struct TestCase<'a> {
536	include_collations: &'a [&'a str],
537	language: DataLocale,	×
538	expected: &'a [&'a str],	×
539	}
540	let cases = [	2✔
541	TestCase {	1✔
542	include_collations: &[],
543	language: locale!("zh").into(),	1✔
544	expected: &["", "stroke", "unihan", "zhuyin"],
545	},
546	TestCase {	1✔
547	include_collations: &["search*"],
548	language: locale!("zh").into(),	1✔
549	expected: &["", "stroke", "unihan", "zhuyin"],
550	},
551	TestCase {	1✔
552	include_collations: &[],
553	language: locale!("ko").into(),	1✔
554	expected: &["", "unihan"],
555	},
556	TestCase {	1✔
557	include_collations: &["search"],
558	language: locale!("ko").into(),	1✔
559	expected: &["", "search", "unihan"],
560	},
561	TestCase {	1✔
562	include_collations: &["searchjl"],
563	language: locale!("ko").into(),	1✔
564	expected: &["", "searchjl", "unihan"],
565	},
566	TestCase {	1✔
567	include_collations: &["search", "searchjl"],
568	language: locale!("ko").into(),	1✔
569	expected: &["", "search", "searchjl", "unihan"],
570	},
571	TestCase {	1✔
572	include_collations: &["search*"],
573	language: locale!("ko").into(),	1✔
574	expected: &["", "search", "searchjl", "unihan"],
575	},
576	TestCase {	1✔
577	include_collations: &[],
578	language: locale!("und").into(),	1✔
579	expected: &["", "emoji", "eor"],
580	},
581	];
582	for cas in cases {	1✔
583	let driver = ExportDriver::new(	16✔
584	[DataLocaleFamily::single(cas.language)],	8✔
585	DeduplicationStrategy::None.into(),	8✔
586	LocaleFallbacker::new_without_data(),	8✔
587	)	8✔
588	.with_additional_collations(cas.include_collations.iter().copied().map(String::from));	8✔
589	let resolved_locales = select_locales_for_marker(	8✔
590	&Provider,
591	icu::collator::provider::CollationTailoringV1::INFO,
592	&driver.requested_families,	8✔
593	&driver.attributes_filters,	8✔
594	false,
595	&driver.fallbacker,	8✔
596	)
597	.unwrap()
598	.into_iter()
599	.map(\|id\| id.marker_attributes.to_string())	27✔
600	.collect::<BTreeSet<_>>();
601	let expected_locales = cas	8✔
602	.expected
603	.iter()
604	.copied()
605	.map(String::from)
606	.collect::<BTreeSet<_>>();
607	assert_eq!(resolved_locales, expected_locales, "{cas:?}");	8✔
608	}	9✔
609	}	2✔

zbraniecki / icu4x / 13958601093

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous