17537320435

Committed 07 Sep 2025 05:00PM UTC coverage: 72.904% (-1.3%) from 74.164%

Build # 17537320435

Build Type

push

github

Committed by

web-flow

Commit Message

Export a zoneinfo64 const for testing (#6916)

This is useful for other crates for testing, and we vendor a copy of
this data already.

It's also easier for us to vendor it since the data is under the same
license. If temporal_rs wishes to vendor it the licensing situation gets
more complicated.

<!--
Thank you for your pull request to ICU4X!

Reminder: try to use [Conventional
Comments](https://conventionalcomments.org/) to make comments clearer.

Please see
https://github.com/unicode-org/icu4x/blob/main/CONTRIBUTING.md for
general
information on contributing to ICU4X.
-->

Coverage Stats

1 of 1 new or added line in 1 file covered. (100.0%)

9526 existing lines in 484 files now uncovered.

60429 of 82889 relevant lines covered (72.9%)

468973.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.59

/components/segmenter/src/complex/mod.rs

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use crate::provider::*;
use crate::{GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed};
use alloc::vec::Vec;
use icu_provider::prelude::*;

mod dictionary;
use dictionary::*;
mod language;
use language::*;
#[cfg(feature = "lstm")]
mod lstm;
#[cfg(feature = "lstm")]
use lstm::*;

#[derive(Debug, Clone)]
#[expect(clippy::large_enum_variant)]
enum DictOrLstm {
    Dict(DataPayload<UCharDictionaryBreakDataV1>),
    #[cfg(feature = "lstm")]
    Lstm(DataPayload<SegmenterLstmAutoV1>),
}

#[derive(Debug, Clone, Copy)]
enum DictOrLstmBorrowed<'data> {
    Dict(&'data UCharDictionaryBreakData<'data>),
    #[cfg(feature = "lstm")]
    Lstm(&'data LstmData<'data>),
}

fn borrow_dictor(dict_or: &DictOrLstm) -> DictOrLstmBorrowed<'_> {
    match dict_or {
        DictOrLstm::Dict(dict) => DictOrLstmBorrowed::Dict(dict.get()),
        #[cfg(feature = "lstm")]
        DictOrLstm::Lstm(lstm) => DictOrLstmBorrowed::Lstm(lstm.get()),
    }
}

fn fromstatic_dictor(dict_or: DictOrLstmBorrowed<'static>) -> DictOrLstm {
    match dict_or {
        DictOrLstmBorrowed::Dict(dict) => DictOrLstm::Dict(DataPayload::from_static_ref(dict)),
        #[cfg(feature = "lstm")]
        DictOrLstmBorrowed::Lstm(lstm) => DictOrLstm::Lstm(DataPayload::from_static_ref(lstm)),
    }
}

#[derive(Debug)]
pub(crate) struct ComplexPayloads {
    grapheme: GraphemeClusterSegmenter,
    my: Option<DictOrLstm>,
    km: Option<DictOrLstm>,
    lo: Option<DictOrLstm>,
    th: Option<DictOrLstm>,
    ja: Option<DataPayload<UCharDictionaryBreakDataV1>>,
}

#[derive(Debug, Clone, Copy)]
pub(crate) struct ComplexPayloadsBorrowed<'data> {
    grapheme: GraphemeClusterSegmenterBorrowed<'data>,
    my: Option<DictOrLstmBorrowed<'data>>,
    km: Option<DictOrLstmBorrowed<'data>>,
    lo: Option<DictOrLstmBorrowed<'data>>,
    th: Option<DictOrLstmBorrowed<'data>>,
    ja: Option<&'data UCharDictionaryBreakData<'data>>,
}

#[cfg(feature = "lstm")]
const MY_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Burmese_");
#[cfg(feature = "lstm")]
const KM_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Khmer_");
#[cfg(feature = "lstm")]
const LO_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Lao_");
#[cfg(feature = "lstm")]
const TH_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Thai_");

const MY_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("burmesedict");
const KM_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("khmerdict");
const LO_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("laodict");
const TH_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("thaidict");
const CJ_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("cjdict");

impl<'data> ComplexPayloadsBorrowed<'data> {
    fn select(&self, language: Language) -> Option<DictOrLstmBorrowed<'data>> {
        const ERR: DataError = DataError::custom("No segmentation model for language");
        match language {
            Language::Burmese => self.my.or_else(|| {
                ERR.with_display_context("my");
                None
            }),
            Language::Khmer => self.km.or_else(|| {
                ERR.with_display_context("km");
                None
            }),
            Language::Lao => self.lo.or_else(|| {
                ERR.with_display_context("lo");
                None
            }),
            Language::Thai => self.th.or_else(|| {
                ERR.with_display_context("th");
                None
            }),
            Language::ChineseOrJapanese => self.ja.map(DictOrLstmBorrowed::Dict).or_else(|| {
                ERR.with_display_context("ja");
                None
            }),
            Language::Unknown => None,
        }
    }
    pub(crate) fn complex_language_segment_str(&self, input: &str) -> Vec<usize> {
        let mut result = Vec::new();
        let mut offset = 0;
        for (slice, lang) in LanguageIterator::new(input) {
            match self.select(lang) {
                Some(DictOrLstmBorrowed::Dict(dict)) => {
                    let seg = DictionarySegmenter::new(dict, self.grapheme);
                    result.extend(seg.segment_str(slice).map(|n| offset + n));
                }
                #[cfg(feature = "lstm")]
                Some(DictOrLstmBorrowed::Lstm(lstm)) => {
                    let seg = LstmSegmenter::new(lstm, self.grapheme);
                    result.extend(seg.segment_str(slice).map(|n| offset + n));
                }
                None => {
                    result.push(offset + slice.len());
                }
            }
            offset += slice.len();
        }
        result
    }
    /// Return UTF-16 segment offset array using dictionary or lstm segmenter.
    pub(crate) fn complex_language_segment_utf16(&self, input: &[u16]) -> Vec<usize> {
        let mut result = Vec::new();
        let mut offset = 0;
        for (slice, lang) in LanguageIteratorUtf16::new(input) {
            match self.select(lang) {
                Some(DictOrLstmBorrowed::Dict(dict)) => {
                    let seg = DictionarySegmenter::new(dict, self.grapheme);
                    result.extend(seg.segment_utf16(slice).map(|n| offset + n));
                }
                #[cfg(feature = "lstm")]
                Some(DictOrLstmBorrowed::Lstm(lstm)) => {
                    let seg = LstmSegmenter::new(lstm, self.grapheme);
                    result.extend(seg.segment_utf16(slice).map(|n| offset + n));
                }
                None => {
                    result.push(offset + slice.len());
                }
            }
            offset += slice.len();
        }
        result
    }
}
impl ComplexPayloadsBorrowed<'static> {
    #[cfg(feature = "lstm")]
    #[cfg(feature = "compiled_data")]
    pub(crate) fn new_lstm() -> Self {
        #[expect(clippy::unwrap_used)]
        // try_load is infallible if the provider only returns `MissingLocale`.
        Self {
            grapheme: GraphemeClusterSegmenter::new(),
            my: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, MY_LSTM)
                .unwrap()
                .map(DictOrLstmBorrowed::Lstm),
            km: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, KM_LSTM)
                .unwrap()
                .map(DictOrLstmBorrowed::Lstm),
            lo: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, LO_LSTM)
                .unwrap()
                .map(DictOrLstmBorrowed::Lstm),
            th: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, TH_LSTM)
                .unwrap()
                .map(DictOrLstmBorrowed::Lstm),
            ja: None,
        }
    }
    #[cfg(feature = "auto")]
    #[cfg(feature = "compiled_data")]
    #[expect(clippy::unwrap_used)]
    pub(crate) fn new_auto() -> Self {
        let mut this = Self::new_lstm();
        this.ja = try_load_static::<SegmenterDictionaryAutoV1, _>(&crate::provider::Baked, CJ_DICT)
            .unwrap();
        this
    }
    #[cfg(feature = "compiled_data")]
    pub(crate) fn new_dict() -> Self {
        #[expect(clippy::unwrap_used)]
        // try_load is infallible if the provider only returns `MissingLocale`.
        Self {
            grapheme: GraphemeClusterSegmenter::new(),
            my: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                MY_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            km: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                KM_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            lo: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                LO_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            th: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                TH_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            ja: try_load_static::<SegmenterDictionaryAutoV1, _>(&crate::provider::Baked, CJ_DICT)
                .unwrap(),
        }
    }

    #[cfg(feature = "compiled_data")]
    pub(crate) fn new_southeast_asian() -> Self {
        #[expect(clippy::unwrap_used)]
        // try_load is infallible if the provider only returns `MissingLocale`.
        Self {
            grapheme: GraphemeClusterSegmenter::new(),
            my: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                MY_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            km: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                KM_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            lo: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                LO_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            th: try_load_static::<SegmenterDictionaryExtendedV1, _>(
                &crate::provider::Baked,
                TH_DICT,
            )
            .unwrap()
            .map(DictOrLstmBorrowed::Dict),
            ja: None,
        }
    }

    pub(crate) fn static_to_owned(self) -> ComplexPayloads {
        ComplexPayloads {
            grapheme: self.grapheme.static_to_owned(),
            my: self.my.map(fromstatic_dictor),
            km: self.km.map(fromstatic_dictor),
            lo: self.lo.map(fromstatic_dictor),
            th: self.th.map(fromstatic_dictor),
            ja: self.ja.map(DataPayload::from_static_ref),
        }
    }
}

impl ComplexPayloads {
    pub(crate) fn as_borrowed(&self) -> ComplexPayloadsBorrowed<'_> {
        ComplexPayloadsBorrowed {
            grapheme: self.grapheme.as_borrowed(),
            my: self.my.as_ref().map(borrow_dictor),
            km: self.km.as_ref().map(borrow_dictor),
            lo: self.lo.as_ref().map(borrow_dictor),
            th: self.th.as_ref().map(borrow_dictor),
            ja: self.ja.as_ref().map(|p| p.get()),
        }
    }

    #[cfg(feature = "lstm")]
    pub(crate) fn try_new_lstm<D>(provider: &D) -> Result<Self, DataError>
    where
        D: DataProvider<SegmenterBreakGraphemeClusterV1>
            + DataProvider<SegmenterLstmAutoV1>
            + ?Sized,
    {
        Ok(Self {
            grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,
            my: try_load::<SegmenterLstmAutoV1, D>(provider, MY_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            km: try_load::<SegmenterLstmAutoV1, D>(provider, KM_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            lo: try_load::<SegmenterLstmAutoV1, D>(provider, LO_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            th: try_load::<SegmenterLstmAutoV1, D>(provider, TH_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            ja: None,
        })
    }

    pub(crate) fn try_new_dict<D>(provider: &D) -> Result<Self, DataError>
    where
        D: DataProvider<SegmenterBreakGraphemeClusterV1>
            + DataProvider<SegmenterDictionaryExtendedV1>
            + DataProvider<SegmenterDictionaryAutoV1>
            + ?Sized,
    {
        Ok(Self {
            grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,
            my: try_load::<SegmenterDictionaryExtendedV1, D>(provider, MY_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            km: try_load::<SegmenterDictionaryExtendedV1, D>(provider, KM_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            lo: try_load::<SegmenterDictionaryExtendedV1, D>(provider, LO_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            th: try_load::<SegmenterDictionaryExtendedV1, D>(provider, TH_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            ja: try_load::<SegmenterDictionaryAutoV1, D>(provider, CJ_DICT)?.map(DataPayload::cast),
        })
    }

    #[cfg(feature = "auto")] // Use by WordSegmenter with "auto" enabled.
    pub(crate) fn try_new_auto<D>(provider: &D) -> Result<Self, DataError>
    where
        D: DataProvider<SegmenterBreakGraphemeClusterV1>
            + DataProvider<SegmenterLstmAutoV1>
            + DataProvider<SegmenterDictionaryAutoV1>
            + ?Sized,
    {
        Ok(Self {
            grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,
            my: try_load::<SegmenterLstmAutoV1, D>(provider, MY_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            km: try_load::<SegmenterLstmAutoV1, D>(provider, KM_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            lo: try_load::<SegmenterLstmAutoV1, D>(provider, LO_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            th: try_load::<SegmenterLstmAutoV1, D>(provider, TH_LSTM)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Lstm),
            ja: try_load::<SegmenterDictionaryAutoV1, D>(provider, CJ_DICT)?.map(DataPayload::cast),
        })
    }

    pub(crate) fn try_new_southeast_asian<D>(provider: &D) -> Result<Self, DataError>
    where
        D: DataProvider<SegmenterDictionaryExtendedV1>
            + DataProvider<SegmenterBreakGraphemeClusterV1>
            + ?Sized,
    {
        Ok(Self {
            grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,
            my: try_load::<SegmenterDictionaryExtendedV1, _>(provider, MY_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            km: try_load::<SegmenterDictionaryExtendedV1, _>(provider, KM_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            lo: try_load::<SegmenterDictionaryExtendedV1, _>(provider, LO_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            th: try_load::<SegmenterDictionaryExtendedV1, _>(provider, TH_DICT)?
                .map(DataPayload::cast)
                .map(DictOrLstm::Dict),
            ja: None,
        })
    }
}
fn try_load<M: DataMarker, P: DataProvider<M> + ?Sized>(
    provider: &P,
    model: &'static DataMarkerAttributes,
) -> Result<Option<DataPayload<M>>, DataError> {
    provider
        .load(DataRequest {
            id: DataIdentifierBorrowed::for_marker_attributes(model),
            metadata: {
                let mut m = DataRequestMetadata::default();
                m.silent = true;
                m.attributes_prefix_match = true;
                m
            },
        })
        .allow_identifier_not_found()
        .map(|r| r.map(|r| r.payload))
}

#[cfg(feature = "compiled_data")]
fn try_load_static<M: DataMarker, P: DataProvider<M> + ?Sized>(
    provider: &P,
    model: &'static DataMarkerAttributes,
) -> Result<Option<&'static <M::DataStruct as yoke::Yokeable<'static>>::Output>, DataError> {
    provider
        .load(DataRequest {
            id: DataIdentifierBorrowed::for_marker_attributes(model),
            metadata: {
                let mut m = DataRequestMetadata::default();
                m.silent = true;
                m.attributes_prefix_match = true;
                m
            },
        })
        .allow_identifier_not_found()
        .map(|r| r.and_then(|r| r.payload.get_static()))
}

#[cfg(test)]
#[cfg(feature = "serde")]
mod tests {
    use super::*;

    #[test]
    fn thai_word_break() {
        const TEST_STR: &str = "ภาษาไทยภาษาไทย";
        let utf16: Vec<u16> = TEST_STR.encode_utf16().collect();

        let lstm = ComplexPayloadsBorrowed::new_lstm();
        let dict = ComplexPayloadsBorrowed::new_dict();

        assert_eq!(
            lstm.complex_language_segment_str(TEST_STR),
            [12, 21, 33, 42]
        );
        assert_eq!(lstm.complex_language_segment_utf16(&utf16), [4, 7, 11, 14]);

        assert_eq!(
            dict.complex_language_segment_str(TEST_STR),
            [12, 21, 33, 42]
        );
        assert_eq!(dict.complex_language_segment_utf16(&utf16), [4, 7, 11, 14]);
    }
}

1	// This file is part of ICU4X. For terms of use, please see the file
2	// called LICENSE at the top level of the ICU4X source tree
3	// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5	use crate::provider::*;
6	use crate::{GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed};
7	use alloc::vec::Vec;
8	use icu_provider::prelude::*;
9
10	mod dictionary;
11	use dictionary::*;
12	mod language;
13	use language::*;
14	#[cfg(feature = "lstm")]
15	mod lstm;
16	#[cfg(feature = "lstm")]
17	use lstm::*;
18
UNCOV 19	#[derive(Debug, Clone)]	×
20	#[expect(clippy::large_enum_variant)]
21	enum DictOrLstm {
22	Dict(DataPayload<UCharDictionaryBreakDataV1>),	16✔
23	#[cfg(feature = "lstm")]
24	Lstm(DataPayload<SegmenterLstmAutoV1>),	12✔
25	}
26
UNCOV 27	#[derive(Debug, Clone, Copy)]	×
28	enum DictOrLstmBorrowed<'data> {
29	Dict(&'data UCharDictionaryBreakData<'data>),	370✔
30	#[cfg(feature = "lstm")]
31	Lstm(&'data LstmData<'data>),	122✔
32	}
33
34	fn borrow_dictor(dict_or: &DictOrLstm) -> DictOrLstmBorrowed<'_> {	28✔
35	match dict_or {	28✔
36	DictOrLstm::Dict(dict) => DictOrLstmBorrowed::Dict(dict.get()),	16✔
37	#[cfg(feature = "lstm")]
38	DictOrLstm::Lstm(lstm) => DictOrLstmBorrowed::Lstm(lstm.get()),	12✔
39	}
40	}	28✔
41
UNCOV 42	fn fromstatic_dictor(dict_or: DictOrLstmBorrowed<'static>) -> DictOrLstm {	×
UNCOV 43	match dict_or {	×
UNCOV 44	DictOrLstmBorrowed::Dict(dict) => DictOrLstm::Dict(DataPayload::from_static_ref(dict)),	×
45	#[cfg(feature = "lstm")]
UNCOV 46	DictOrLstmBorrowed::Lstm(lstm) => DictOrLstm::Lstm(DataPayload::from_static_ref(lstm)),	×
47	}
UNCOV 48	}	×
49
UNCOV 50	#[derive(Debug)]	×
51	pub(crate) struct ComplexPayloads {
UNCOV 52	grapheme: GraphemeClusterSegmenter,	×
UNCOV 53	my: Option<DictOrLstm>,	×
UNCOV 54	km: Option<DictOrLstm>,	×
UNCOV 55	lo: Option<DictOrLstm>,	×
UNCOV 56	th: Option<DictOrLstm>,	×
UNCOV 57	ja: Option<DataPayload<UCharDictionaryBreakDataV1>>,	×
58	}
59
60	#[derive(Debug, Clone, Copy)]	×
61	pub(crate) struct ComplexPayloadsBorrowed<'data> {
62	grapheme: GraphemeClusterSegmenterBorrowed<'data>,	×
63	my: Option<DictOrLstmBorrowed<'data>>,
64	km: Option<DictOrLstmBorrowed<'data>>,	×
65	lo: Option<DictOrLstmBorrowed<'data>>,	×
66	th: Option<DictOrLstmBorrowed<'data>>,	×
UNCOV 67	ja: Option<&'data UCharDictionaryBreakData<'data>>,	×
68	}
69
70	#[cfg(feature = "lstm")]
71	const MY_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Burmese_");
72	#[cfg(feature = "lstm")]
73	const KM_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Khmer_");
74	#[cfg(feature = "lstm")]
75	const LO_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Lao_");
76	#[cfg(feature = "lstm")]
77	const TH_LSTM: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("Thai_");
78
79	const MY_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("burmesedict");
80	const KM_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("khmerdict");
81	const LO_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("laodict");
82	const TH_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("thaidict");
83	const CJ_DICT: &DataMarkerAttributes = DataMarkerAttributes::from_str_or_panic("cjdict");
84
85	impl<'data> ComplexPayloadsBorrowed<'data> {
86	fn select(&self, language: Language) -> Option<DictOrLstmBorrowed<'data>> {	78✔
87	const ERR: DataError = DataError::custom("No segmentation model for language");
88	match language {	78✔
89	Language::Burmese => self.my.or_else(\|\| {	5✔
UNCOV 90	ERR.with_display_context("my");	×
UNCOV 91	None	×
UNCOV 92	}),	×
93	Language::Khmer => self.km.or_else(\|\| {	6✔
UNCOV 94	ERR.with_display_context("km");	×
UNCOV 95	None	×
UNCOV 96	}),	×
97	Language::Lao => self.lo.or_else(\|\| {	6✔
UNCOV 98	ERR.with_display_context("lo");	×
UNCOV 99	None	×
UNCOV 100	}),	×
101	Language::Thai => self.th.or_else(\|\| {	49✔
UNCOV 102	ERR.with_display_context("th");	×
UNCOV 103	None	×
UNCOV 104	}),	×
105	Language::ChineseOrJapanese => self.ja.map(DictOrLstmBorrowed::Dict).or_else(\|\| {	13✔
106	ERR.with_display_context("ja");	1✔
107	None	1✔
108	}),	1✔
109	Language::Unknown => None,	×
110	}
111	}	78✔
112	pub(crate) fn complex_language_segment_str(&self, input: &str) -> Vec<usize> {	42✔
113	let mut result = Vec::new();	42✔
114	let mut offset = 0;	42✔
115	for (slice, lang) in LanguageIterator::new(input) {	84✔
116	match self.select(lang) {	42✔
117	Some(DictOrLstmBorrowed::Dict(dict)) => {	17✔
118	let seg = DictionarySegmenter::new(dict, self.grapheme);	17✔
119	result.extend(seg.segment_str(slice).map(\|n\| offset + n));	71✔
120	}
121	#[cfg(feature = "lstm")]
122	Some(DictOrLstmBorrowed::Lstm(lstm)) => {	23✔
123	let seg = LstmSegmenter::new(lstm, self.grapheme);	23✔
124	result.extend(seg.segment_str(slice).map(\|n\| offset + n));	107✔
125	}
126	None => {
127	result.push(offset + slice.len());	1✔
128	}
129	}
130	offset += slice.len();	42✔
131	}
132	result	38✔
133	}	38✔
134	/// Return UTF-16 segment offset array using dictionary or lstm segmenter.
135	pub(crate) fn complex_language_segment_utf16(&self, input: &[u16]) -> Vec<usize> {	36✔
136	let mut result = Vec::new();	36✔
137	let mut offset = 0;	36✔
138	for (slice, lang) in LanguageIteratorUtf16::new(input) {	72✔
139	match self.select(lang) {	36✔
140	Some(DictOrLstmBorrowed::Dict(dict)) => {	12✔
141	let seg = DictionarySegmenter::new(dict, self.grapheme);	12✔
142	result.extend(seg.segment_utf16(slice).map(\|n\| offset + n));	53✔
143	}
144	#[cfg(feature = "lstm")]
145	Some(DictOrLstmBorrowed::Lstm(lstm)) => {	24✔
146	let seg = LstmSegmenter::new(lstm, self.grapheme);	24✔
147	result.extend(seg.segment_utf16(slice).map(\|n\| offset + n));	110✔
148	}
149	None => {
UNCOV 150	result.push(offset + slice.len());	×
151	}
152	}
153	offset += slice.len();	36✔
154	}
155	result	34✔
156	}	34✔
157	}
158	impl ComplexPayloadsBorrowed<'static> {
159	#[cfg(feature = "lstm")]
160	#[cfg(feature = "compiled_data")]
161	pub(crate) fn new_lstm() -> Self {	31✔
162	#[expect(clippy::unwrap_used)]
163	// try_load is infallible if the provider only returns `MissingLocale`.
164	Self {	31✔
165	grapheme: GraphemeClusterSegmenter::new(),	31✔
166	my: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, MY_LSTM)	31✔
167	.unwrap()
168	.map(DictOrLstmBorrowed::Lstm),
169	km: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, KM_LSTM)	31✔
170	.unwrap()
171	.map(DictOrLstmBorrowed::Lstm),
172	lo: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, LO_LSTM)	31✔
173	.unwrap()
174	.map(DictOrLstmBorrowed::Lstm),
175	th: try_load_static::<SegmenterLstmAutoV1, _>(&crate::provider::Baked, TH_LSTM)	31✔
176	.unwrap()
177	.map(DictOrLstmBorrowed::Lstm),
178	ja: None,	31✔
179	}
180	}	31✔
181	#[cfg(feature = "auto")]
182	#[cfg(feature = "compiled_data")]
183	#[expect(clippy::unwrap_used)]
184	pub(crate) fn new_auto() -> Self {	15✔
185	let mut this = Self::new_lstm();	15✔
186	this.ja = try_load_static::<SegmenterDictionaryAutoV1, _>(&crate::provider::Baked, CJ_DICT)	15✔
187	.unwrap();
188	this	15✔
189	}	15✔
190	#[cfg(feature = "compiled_data")]
191	pub(crate) fn new_dict() -> Self {	4✔
192	#[expect(clippy::unwrap_used)]
193	// try_load is infallible if the provider only returns `MissingLocale`.
194	Self {	4✔
195	grapheme: GraphemeClusterSegmenter::new(),	4✔
196	my: try_load_static::<SegmenterDictionaryExtendedV1, _>(	4✔
197	&crate::provider::Baked,
198	MY_DICT,
199	)
200	.unwrap()
201	.map(DictOrLstmBorrowed::Dict),
202	km: try_load_static::<SegmenterDictionaryExtendedV1, _>(	4✔
203	&crate::provider::Baked,
204	KM_DICT,
205	)
206	.unwrap()
207	.map(DictOrLstmBorrowed::Dict),
208	lo: try_load_static::<SegmenterDictionaryExtendedV1, _>(	4✔
209	&crate::provider::Baked,
210	LO_DICT,
211	)
212	.unwrap()
213	.map(DictOrLstmBorrowed::Dict),
214	th: try_load_static::<SegmenterDictionaryExtendedV1, _>(	4✔
215	&crate::provider::Baked,
216	TH_DICT,
217	)
218	.unwrap()
219	.map(DictOrLstmBorrowed::Dict),
220	ja: try_load_static::<SegmenterDictionaryAutoV1, _>(&crate::provider::Baked, CJ_DICT)	4✔
221	.unwrap(),
222	}
223	}	4✔
224
225	#[cfg(feature = "compiled_data")]
226	pub(crate) fn new_southeast_asian() -> Self {	90✔
227	#[expect(clippy::unwrap_used)]
228	// try_load is infallible if the provider only returns `MissingLocale`.
229	Self {	90✔
230	grapheme: GraphemeClusterSegmenter::new(),	90✔
231	my: try_load_static::<SegmenterDictionaryExtendedV1, _>(	90✔
232	&crate::provider::Baked,
233	MY_DICT,
234	)
235	.unwrap()
236	.map(DictOrLstmBorrowed::Dict),
237	km: try_load_static::<SegmenterDictionaryExtendedV1, _>(	90✔
238	&crate::provider::Baked,
239	KM_DICT,
240	)
241	.unwrap()
242	.map(DictOrLstmBorrowed::Dict),
243	lo: try_load_static::<SegmenterDictionaryExtendedV1, _>(	90✔
244	&crate::provider::Baked,
245	LO_DICT,
246	)
247	.unwrap()
248	.map(DictOrLstmBorrowed::Dict),
249	th: try_load_static::<SegmenterDictionaryExtendedV1, _>(	90✔
250	&crate::provider::Baked,
251	TH_DICT,
252	)
253	.unwrap()
254	.map(DictOrLstmBorrowed::Dict),
255	ja: None,	90✔
256	}
257	}	90✔
258
UNCOV 259	pub(crate) fn static_to_owned(self) -> ComplexPayloads {	×
UNCOV 260	ComplexPayloads {	×
UNCOV 261	grapheme: self.grapheme.static_to_owned(),	×
UNCOV 262	my: self.my.map(fromstatic_dictor),	×
UNCOV 263	km: self.km.map(fromstatic_dictor),	×
UNCOV 264	lo: self.lo.map(fromstatic_dictor),	×
UNCOV 265	th: self.th.map(fromstatic_dictor),	×
UNCOV 266	ja: self.ja.map(DataPayload::from_static_ref),	×
UNCOV 267	}	×
UNCOV 268	}	×
269	}
270
271	impl ComplexPayloads {
272	pub(crate) fn as_borrowed(&self) -> ComplexPayloadsBorrowed<'_> {	7✔
273	ComplexPayloadsBorrowed {	7✔
274	grapheme: self.grapheme.as_borrowed(),	7✔
275	my: self.my.as_ref().map(borrow_dictor),	7✔
276	km: self.km.as_ref().map(borrow_dictor),	7✔
277	lo: self.lo.as_ref().map(borrow_dictor),	7✔
278	th: self.th.as_ref().map(borrow_dictor),	7✔
279	ja: self.ja.as_ref().map(\|p\| p.get()),	12✔
280	}
281	}	7✔
282
283	#[cfg(feature = "lstm")]
284	pub(crate) fn try_new_lstm<D>(provider: &D) -> Result<Self, DataError>	1✔
285	where
286	D: DataProvider<SegmenterBreakGraphemeClusterV1>
287	+ DataProvider<SegmenterLstmAutoV1>
288	+ ?Sized,
289	{
290	Ok(Self {	1✔
291	grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,	1✔
292	my: try_load::<SegmenterLstmAutoV1, D>(provider, MY_LSTM)?	1✔
293	.map(DataPayload::cast)
294	.map(DictOrLstm::Lstm),
295	km: try_load::<SegmenterLstmAutoV1, D>(provider, KM_LSTM)?	1✔
296	.map(DataPayload::cast)
297	.map(DictOrLstm::Lstm),
298	lo: try_load::<SegmenterLstmAutoV1, D>(provider, LO_LSTM)?	1✔
299	.map(DataPayload::cast)
300	.map(DictOrLstm::Lstm),
301	th: try_load::<SegmenterLstmAutoV1, D>(provider, TH_LSTM)?	1✔
302	.map(DataPayload::cast)
303	.map(DictOrLstm::Lstm),
304	ja: None,	1✔
UNCOV 305	})	×
306	}	1✔
307
308	pub(crate) fn try_new_dict<D>(provider: &D) -> Result<Self, DataError>	3✔
309	where
310	D: DataProvider<SegmenterBreakGraphemeClusterV1>
311	+ DataProvider<SegmenterDictionaryExtendedV1>
312	+ DataProvider<SegmenterDictionaryAutoV1>
313	+ ?Sized,
314	{
315	Ok(Self {	3✔
316	grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,	3✔
317	my: try_load::<SegmenterDictionaryExtendedV1, D>(provider, MY_DICT)?	3✔
318	.map(DataPayload::cast)
319	.map(DictOrLstm::Dict),
320	km: try_load::<SegmenterDictionaryExtendedV1, D>(provider, KM_DICT)?	3✔
321	.map(DataPayload::cast)
322	.map(DictOrLstm::Dict),
323	lo: try_load::<SegmenterDictionaryExtendedV1, D>(provider, LO_DICT)?	3✔
324	.map(DataPayload::cast)
325	.map(DictOrLstm::Dict),
326	th: try_load::<SegmenterDictionaryExtendedV1, D>(provider, TH_DICT)?	3✔
327	.map(DataPayload::cast)
328	.map(DictOrLstm::Dict),
329	ja: try_load::<SegmenterDictionaryAutoV1, D>(provider, CJ_DICT)?.map(DataPayload::cast),	3✔
UNCOV 330	})	×
331	}	3✔
332
333	#[cfg(feature = "auto")] // Use by WordSegmenter with "auto" enabled.
334	pub(crate) fn try_new_auto<D>(provider: &D) -> Result<Self, DataError>	2✔
335	where
336	D: DataProvider<SegmenterBreakGraphemeClusterV1>
337	+ DataProvider<SegmenterLstmAutoV1>
338	+ DataProvider<SegmenterDictionaryAutoV1>
339	+ ?Sized,
340	{
341	Ok(Self {	2✔
342	grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,	2✔
343	my: try_load::<SegmenterLstmAutoV1, D>(provider, MY_LSTM)?	2✔
344	.map(DataPayload::cast)
345	.map(DictOrLstm::Lstm),
346	km: try_load::<SegmenterLstmAutoV1, D>(provider, KM_LSTM)?	2✔
347	.map(DataPayload::cast)
348	.map(DictOrLstm::Lstm),
349	lo: try_load::<SegmenterLstmAutoV1, D>(provider, LO_LSTM)?	2✔
350	.map(DataPayload::cast)
351	.map(DictOrLstm::Lstm),
352	th: try_load::<SegmenterLstmAutoV1, D>(provider, TH_LSTM)?	2✔
353	.map(DataPayload::cast)
354	.map(DictOrLstm::Lstm),
355	ja: try_load::<SegmenterDictionaryAutoV1, D>(provider, CJ_DICT)?.map(DataPayload::cast),	2✔
UNCOV 356	})	×
357	}	2✔
358
359	pub(crate) fn try_new_southeast_asian<D>(provider: &D) -> Result<Self, DataError>	1✔
360	where
361	D: DataProvider<SegmenterDictionaryExtendedV1>
362	+ DataProvider<SegmenterBreakGraphemeClusterV1>
363	+ ?Sized,
364	{
365	Ok(Self {	1✔
366	grapheme: GraphemeClusterSegmenter::try_new_unstable(provider)?,	1✔
367	my: try_load::<SegmenterDictionaryExtendedV1, _>(provider, MY_DICT)?	1✔
368	.map(DataPayload::cast)
369	.map(DictOrLstm::Dict),
370	km: try_load::<SegmenterDictionaryExtendedV1, _>(provider, KM_DICT)?	1✔
371	.map(DataPayload::cast)
372	.map(DictOrLstm::Dict),
373	lo: try_load::<SegmenterDictionaryExtendedV1, _>(provider, LO_DICT)?	1✔
374	.map(DataPayload::cast)
375	.map(DictOrLstm::Dict),
376	th: try_load::<SegmenterDictionaryExtendedV1, _>(provider, TH_DICT)?	1✔
377	.map(DataPayload::cast)
378	.map(DictOrLstm::Dict),
379	ja: None,	1✔
UNCOV 380	})	×
381	}	1✔
382	}
383	fn try_load<M: DataMarker, P: DataProvider<M> + ?Sized>(	33✔
384	provider: &P,
385	model: &'static DataMarkerAttributes,
386	) -> Result<Option<DataPayload<M>>, DataError> {
387	provider	33✔
388	.load(DataRequest {	33✔
389	id: DataIdentifierBorrowed::for_marker_attributes(model),	33✔
390	metadata: {
391	let mut m = DataRequestMetadata::default();	33✔
392	m.silent = true;	33✔
393	m.attributes_prefix_match = true;	33✔
394	m	33✔
395	},
396	})
397	.allow_identifier_not_found()
398	.map(\|r\| r.map(\|r\| r.payload))	66✔
399	}	33✔
400
401	#[cfg(feature = "compiled_data")]
402	fn try_load_static<M: DataMarker, P: DataProvider<M> + ?Sized>(	512✔
403	provider: &P,
404	model: &'static DataMarkerAttributes,
405	) -> Result<Option<&'static <M::DataStruct as yoke::Yokeable<'static>>::Output>, DataError> {
406	provider	512✔
407	.load(DataRequest {	512✔
408	id: DataIdentifierBorrowed::for_marker_attributes(model),	512✔
409	metadata: {
410	let mut m = DataRequestMetadata::default();	512✔
411	m.silent = true;	512✔
412	m.attributes_prefix_match = true;	512✔
413	m	512✔
414	},
415	})
416	.allow_identifier_not_found()
417	.map(\|r\| r.and_then(\|r\| r.payload.get_static()))	1,030✔
418	}	512✔
419
420	#[cfg(test)]
421	#[cfg(feature = "serde")]
422	mod tests {
423	use super::*;
424
425	#[test]
426	fn thai_word_break() {	2✔
427	const TEST_STR: &str = "ภาษาไทยภาษาไทย";
428	let utf16: Vec<u16> = TEST_STR.encode_utf16().collect();	1✔
429
430	let lstm = ComplexPayloadsBorrowed::new_lstm();	1✔
431	let dict = ComplexPayloadsBorrowed::new_dict();	1✔
432
433	assert_eq!(	2✔
434	lstm.complex_language_segment_str(TEST_STR),	1✔
435	[12, 21, 33, 42]
436	);
437	assert_eq!(lstm.complex_language_segment_utf16(&utf16), [4, 7, 11, 14]);	2✔
438
439	assert_eq!(	2✔
440	dict.complex_language_segment_str(TEST_STR),	1✔
441	[12, 21, 33, 42]
442	);
443	assert_eq!(dict.complex_language_segment_utf16(&utf16), [4, 7, 11, 14]);	2✔
444	}	2✔
445	}

zbraniecki / icu4x / 17537320435

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous