12076846628

Committed 29 Nov 2024 12:11AM UTC coverage: 81.241% (-1.6%) from 82.811%

Build # 12076846628

Build Type

push

github

Committed by

Aloso

Commit Message

chore: try linking PCRE2 statically

Run Details

4296 of 5288 relevant lines covered (81.24%)

406362.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.51

/pomsky-lib/src/exprs/char_class.rs

//! Implements _character classes_. The analogue in the regex world are
//! [character classes](https://www.regular-expressions.info/charclass.html),
//! [shorthand character classes](https://www.regular-expressions.info/shorthand.html),
//! [non-printable characters](https://www.regular-expressions.info/nonprint.html),
//! [Unicode categories/scripts/blocks](https://www.regular-expressions.info/unicode.html#category),
//! [POSIX classes](https://www.regular-expressions.info/posixbrackets.html#class) and the
//! [dot](https://www.regular-expressions.info/dot.html).
//!
//! All kinds of character classes mentioned above require `[` square brackets
//! `]` in Pomsky. A character class can be negated by putting the keyword `not`
//! after the opening bracket. For example, `![.]` compiles to `\n`.
//!
//! ## Items
//!
//! A character class can contain multiple _items_, which can be
//!
//! - A __code point__, e.g. `['a']` or `[U+107]`
//!
//!   - This includes [non-printable characters](https://www.regular-expressions.info/nonprint.html).\
//!     Supported are `[n]`, `[r]`, `[t]`, `[a]`, `[e]` and `[f]`.
//!
//! - A __range of code points__. For example, `[U+10 - U+200]` matches any code
//!   point P where `U+10 ≤ P ≤ U+200`
//!
//! - A __named character class__, which can be one of
//!
//!   - a [shorthand character class](https://www.regular-expressions.info/shorthand.html).\
//!     Supported are `[w]`, `[d]`, `[s]`, `[h]`, `[v]` and `[R]`.
//!
//!   - a [POSIX class](https://www.regular-expressions.info/posixbrackets.html#class).\
//!     Supported are `[ascii_alnum]`, `[ascii_alpha]`, `[ascii]`,
//!     `[ascii_blank]`, `[ascii_cntrl]`, `[ascii_digit]`, `[ascii_graph]`,
//!     `[ascii_lower]`, `[ascii_print]`, `[ascii_punct]`, ´ `[ascii_space]`,
//!     `[ascii_upper]`, `[ascii_word]` and `[ascii_xdigit]`.\ _Note_: POSIX
//!     classes are not Unicode aware!\ _Note_: They're converted to ranges,
//!     e.g. `[ascii_alpha]` = `[a-zA-Z]`.
//!
//!   - a [Unicode category, script or block](https://www.regular-expressions.info/unicode.html#category).\
//!     For example: `[Letter]` compiles to `\p{Letter}`. Pomsky currently
//!     treats any uppercase identifier except `R` as Unicode class.
//!
//! ## Compilation
//!
//! When a character class contains only a single item (e.g. `[w]`), the
//! character class is "flattened":
//!
//! - `['a']` = `a`
//! - `[w]` = `\w`
//! - `[Letter]` = `\p{Letter}`
//!
//! When there is more than one item or a range (e.g. `['a'-'z' '!']`), a regex
//! character class is created:
//!
//! - `['a'-'z' '!']` = `[a-z!]`
//! - `[w e Punctuation]` = `[\w\e\p{Punctuation}]`
//!
//! ### Negation
//!
//! Negation is implemented as follows:
//!
//! - Ranges and chars such as `!['a'-'z' '!' e]` are wrapped in a negative
//!   character class, e.g. `[^a-z!\e]`.
//!
//! - The `h`, `v` and `R` shorthands are also wrapped in a negative character
//!   class.
//!
//! - The `w`, `d` and `s` shorthands are negated by making them uppercase
//!   (`![w]` = `\W`), except when there is more than one item in the class
//!   (`![w '-']` = `[^\w\-]`)
//!
//! - `w`, `s`, `d` and Unicode categories/scripts/blocks can be negated
//!   individually _within a character class_, e.g. `[s !s]` = `[\s\S]`,
//!   `![!Latin 'a']` = `[^\P{Latin}a]`.
//!
//!   When a negated character class only contains 1 item, which is also
//!   negated, the class is   removed and the negations cancel each other out:
//!   `![!w]` = `\w`, `![!L]` = `\p{L}`.

use std::fmt;

use crate::{
    compile::{CompileResult, CompileState},
    diagnose::{CompileError, CompileErrorKind, Feature},
    exprs::literal,
    options::{CompileOptions, RegexFlavor},
    regex::{Regex, RegexProperty, RegexShorthand},
    unicode_set::UnicodeSet,
};

use pomsky_syntax::{
    exprs::{
        Category, CharClass, CodeBlock, GroupItem, GroupName, OtherProperties, Script,
        ScriptExtension,
    },
    Span,
};

use super::Compile;

impl Compile for CharClass {
    fn compile(&self, options: CompileOptions, _state: &mut CompileState<'_>) -> CompileResult {
        // when single, a `[!w]` can be turned into `![w]`
        let is_single = self.inner.len() == 1;
        let mut group_negative = false;

        let mut set = UnicodeSet::new();
        for item in &self.inner {
            match *item {
                GroupItem::Char(c) => {
                    if !is_single {
                        validate_char_in_class(c, options.flavor, self.span)?;
                    }
                    set.add_char(c)
                }
                GroupItem::Range { first, last } => {
                    validate_char_in_class(first, options.flavor, self.span)?;
                    validate_char_in_class(last, options.flavor, self.span)?;
                    set.add_range(first..=last);
                }
                GroupItem::Named { name, negative, span } => {
                    if self.unicode_aware {
                        named_class_to_regex_unicode(
                            name,
                            negative,
                            &mut group_negative,
                            is_single,
                            options.flavor,
                            span,
                            &mut set,
                        )?;
                    } else {
                        named_class_to_regex_ascii(name, negative, options.flavor, span, &mut set)?;
                    }
                }
            }
        }

        // this makes it possible to use code points outside the BMP in .NET,
        // as long as there is only one in the character set
        if let Some(only_char) = set.try_into_char() {
            return Ok(Regex::Literal(only_char.to_string()));
        }

        Ok(Regex::CharSet(RegexCharSet { negative: group_negative, set }))
    }
}

fn validate_char_in_class(char: char, flavor: RegexFlavor, span: Span) -> Result<(), CompileError> {
    if flavor == RegexFlavor::DotNet && char > '\u{FFFF}' {
        Err(CompileErrorKind::Unsupported(Feature::LargeCodePointInCharClass(char), flavor)
            .at(span))
    } else {
        Ok(())
    }
}

pub(crate) fn check_char_class_empty(
    char_set: &RegexCharSet,
    span: Span,
) -> Result<(), CompileError> {
    if char_set.negative {
        if let Some((group1, group2)) = char_set.set.full_props() {
            return Err(CompileErrorKind::EmptyClassNegated { group1, group2 }.at(span));
        }
    }
    Ok(())
}

pub fn is_ascii_only_in_flavor(group: GroupName, flavor: RegexFlavor) -> bool {
    match flavor {
        RegexFlavor::JavaScript => matches!(group, GroupName::Word | GroupName::Digit),
        RegexFlavor::RE2 => matches!(group, GroupName::Word | GroupName::Digit | GroupName::Space),
        _ => false,
    }
}

fn named_class_to_regex_ascii(
    group: GroupName,
    negative: bool,
    flavor: RegexFlavor,
    span: Span,
    set: &mut UnicodeSet,
) -> Result<(), CompileError> {
    // In JS, \W and \D can be used for negation because they're ascii-only
    // Same goes for \W, \D and \S in RE2
    if negative && !is_ascii_only_in_flavor(group, flavor) {
        return Err(CompileErrorKind::NegativeShorthandInAsciiMode.at(span));
    }

    match group {
        GroupName::Word => {
            if let RegexFlavor::JavaScript | RegexFlavor::RE2 = flavor {
                let s = if negative { RegexShorthand::NotWord } else { RegexShorthand::Word };
                set.add_prop(RegexCharSetItem::Shorthand(s));
            } else {
                // we already checked above if negative
                set.add_range('a'..='z');
                set.add_range('A'..='Z');
                set.add_range('0'..='9');
                set.add_char('_');
            }
        }
        GroupName::Digit => {
            if let RegexFlavor::JavaScript | RegexFlavor::RE2 = flavor {
                let s = if negative { RegexShorthand::NotDigit } else { RegexShorthand::Digit };
                set.add_prop(RegexCharSetItem::Shorthand(s));
            } else {
                // we already checked above if negative
                set.add_range('0'..='9');
            }
        }
        GroupName::Space => {
            if let RegexFlavor::RE2 = flavor {
                let s = if negative { RegexShorthand::NotSpace } else { RegexShorthand::Space };
                set.add_prop(RegexCharSetItem::Shorthand(s));
            } else {
                set.add_char(' ');
                set.add_range('\x09'..='\x0D'); // \t\n\v\f\r
            }
        }
        GroupName::HorizSpace => set.add_char('\t'),
        GroupName::VertSpace => set.add_range('\x0A'..='\x0D'),
        _ => return Err(CompileErrorKind::UnicodeInAsciiMode.at(span)),
    }
    Ok(())
}

fn named_class_to_regex_unicode(
    group: GroupName,
    negative: bool,
    group_negative: &mut bool,
    is_single: bool,
    flavor: RegexFlavor,
    span: Span,
    set: &mut UnicodeSet,
) -> Result<(), CompileError> {
    match group {
        GroupName::Word => {
            if flavor == RegexFlavor::RE2 {
                return Err(CompileErrorKind::Unsupported(Feature::ShorthandW, flavor).at(span));
            } else if flavor == RegexFlavor::JavaScript {
                if negative {
                    if is_single {
                        *group_negative ^= true;
                    } else {
                        return Err(CompileErrorKind::Unsupported(
                            Feature::NegativeShorthandW,
                            flavor,
                        )
                        .at(span));
                    }
                }
                set.add_prop(
                    RegexProperty::Other(OtherProperties::Alphabetic).negative_item(false),
                );
                set.add_prop(RegexProperty::Category(Category::Mark).negative_item(false));
                set.add_prop(
                    RegexProperty::Category(Category::Decimal_Number).negative_item(false),
                );
                set.add_prop(
                    RegexProperty::Category(Category::Connector_Punctuation).negative_item(false),
                );
            } else {
                let s = if negative { RegexShorthand::NotWord } else { RegexShorthand::Word };
                set.add_prop(RegexCharSetItem::Shorthand(s));
            }
        }
        GroupName::Digit => {
            if matches!(flavor, RegexFlavor::JavaScript | RegexFlavor::RE2) {
                set.add_prop(
                    RegexProperty::Category(Category::Decimal_Number).negative_item(negative),
                );
            } else {
                let s = if negative { RegexShorthand::NotDigit } else { RegexShorthand::Digit };
                set.add_prop(RegexCharSetItem::Shorthand(s));
            }
        }

        GroupName::Space => {
            if flavor == RegexFlavor::RE2 {
                if negative {
                    if is_single {
                        *group_negative ^= true;
                    } else {
                        return Err(CompileErrorKind::Unsupported(
                            Feature::NegativeShorthandS,
                            flavor,
                        )
                        .at(span));
                    }
                }

                // [ \f\n\r\t\u000b\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]
                set.add_prop(RegexCharSetItem::Shorthand(RegexShorthand::Space));
                set.add_char('\x0b');
                set.add_char('\u{a0}');
                set.add_char('\u{1680}');
                set.add_range('\u{2000}'..='\u{200a}');
                set.add_range('\u{2028}'..='\u{2029}');
                set.add_char('\u{202f}');
                set.add_char('\u{205f}');
                set.add_char('\u{3000}');
                set.add_char('\u{feff}');
            } else {
                set.add_prop(RegexCharSetItem::Shorthand(if negative {
                    RegexShorthand::NotSpace
                } else {
                    RegexShorthand::Space
                }))
            }
        }

        GroupName::HorizSpace | GroupName::VertSpace if negative => {
            return Err(CompileErrorKind::NegatedHorizVertSpace.at(span));
        }

        GroupName::HorizSpace | GroupName::VertSpace
            if matches!(flavor, RegexFlavor::Pcre | RegexFlavor::Java) =>
        {
            set.add_prop(RegexCharSetItem::Shorthand(if group == GroupName::HorizSpace {
                RegexShorthand::HorizSpace
            } else {
                RegexShorthand::VertSpace
            }));
        }
        GroupName::HorizSpace => {
            set.add_char('\t');
            if flavor == RegexFlavor::Python {
                return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));
            } else {
                set.add_prop(
                    RegexProperty::Category(Category::Space_Separator).negative_item(false),
                );
            }
        }
        GroupName::VertSpace => {
            set.add_range('\x0A'..='\x0D');
            set.add_char('\u{85}');
            set.add_char('\u{2028}');
            set.add_char('\u{2029}');
        }

        _ if flavor == RegexFlavor::Python => {
            return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));
        }
        GroupName::Category(c) => {
            if let (RegexFlavor::Rust, Category::Surrogate)
            | (RegexFlavor::DotNet | RegexFlavor::RE2, Category::Cased_Letter) = (flavor, c)
            {
                return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));
            }
            set.add_prop(RegexProperty::Category(c).negative_item(negative));
        }
        GroupName::Script(s, e) => {
            if flavor == RegexFlavor::DotNet {
                return Err(CompileErrorKind::Unsupported(Feature::UnicodeScript, flavor).at(span));
            }
            if let (RegexFlavor::Ruby, Script::Kawi | Script::Nag_Mundari)
            | (RegexFlavor::Rust, Script::Unknown) = (flavor, s)
            {
                return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));
            }

            let set_extensions = match e {
                ScriptExtension::Yes => match flavor {
                    RegexFlavor::Rust | RegexFlavor::Pcre | RegexFlavor::JavaScript => {
                        ScriptExtension::Yes
                    }
                    RegexFlavor::Java
                    | RegexFlavor::DotNet
                    | RegexFlavor::Ruby
                    | RegexFlavor::Python
                    | RegexFlavor::RE2 => {
                        return Err(CompileErrorKind::Unsupported(
                            Feature::ScriptExtensions,
                            flavor,
                        )
                        .at(span))
                    }
                },
                ScriptExtension::No => match flavor {
                    // PCRE is currently the only flavor when `\p{Greek}` is the same as `\p{scx=Greek}`
                    RegexFlavor::Pcre => ScriptExtension::No,
                    _ => ScriptExtension::Unspecified,
                },
                _ => ScriptExtension::Unspecified,
            };

            set.add_prop(RegexProperty::Script(s, set_extensions).negative_item(negative));
        }
        GroupName::CodeBlock(b) => match flavor {
            RegexFlavor::DotNet | RegexFlavor::Java | RegexFlavor::Ruby => {
                match (flavor, b) {
                    (RegexFlavor::Java, CodeBlock::No_Block)
                    | (
                        // These should work since Oniguruma updated to Unicode 15.1
                        // ... but our C bindings for Oniguruma are unmaintained!
                        RegexFlavor::Ruby,
                        CodeBlock::Arabic_Extended_C
                        | CodeBlock::CJK_Unified_Ideographs_Extension_H
                        | CodeBlock::Cyrillic_Extended_D
                        | CodeBlock::Devanagari_Extended_A
                        | CodeBlock::Kaktovik_Numerals,
                    ) => {
                        return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));
                    }
                    (RegexFlavor::DotNet, _) => {
                        let dotnet_name = b.as_str().replace("_And_", "_and_").replace('_', "");
                        if pomsky_syntax::blocks_supported_in_dotnet()
                            .binary_search(&dotnet_name.as_str())
                            .is_err()
                        {
                            return Err(
                                CompileErrorKind::unsupported_specific_prop_in(flavor).at(span)
                            );
                        }
                    }
                    _ => {}
                }

                set.add_prop(RegexProperty::Block(b).negative_item(negative));
            }
            _ => return Err(CompileErrorKind::Unsupported(Feature::UnicodeBlock, flavor).at(span)),
        },
        GroupName::OtherProperties(o) => {
            use OtherProperties as OP;
            use RegexFlavor as RF;

            if let RF::JavaScript | RF::Rust | RF::Pcre | RF::Ruby = flavor {
                match (flavor, o) {
                    (RF::JavaScript, _) => {}
                    (_, OP::Changes_When_NFKC_Casefolded)
                    | (RF::Pcre, OP::Assigned)
                    | (RF::Ruby, OP::Bidi_Mirrored) => {
                        return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));
                    }
                    _ => {}
                }
                set.add_prop(RegexProperty::Other(o).negative_item(negative));
            } else {
                return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));
            }
        }
    }
    Ok(())
}

#[cfg_attr(feature = "dbg", derive(Debug))]
#[derive(Default)]
pub(crate) struct RegexCharSet {
    pub(crate) negative: bool,
    pub(crate) set: UnicodeSet,
}

impl RegexCharSet {
    pub(crate) fn new(items: UnicodeSet) -> Self {
        Self { negative: false, set: items }
    }

    pub(crate) fn negate(mut self) -> Self {
        self.negative = !self.negative;
        self
    }

    pub(crate) fn codegen(&self, buf: &mut String, flavor: RegexFlavor) {
        if self.set.len() == 1 {
            if let Some(range) = self.set.ranges().next() {
                let (first, last) = range.as_chars();
                if first == last && !self.negative {
                    return literal::codegen_char_esc(first, buf, flavor);
                }
            } else if let Some(prop) = self.set.props().next() {
                match prop {
                    RegexCharSetItem::Shorthand(s) => {
                        let shorthand = if self.negative { s.negate() } else { Some(s) };
                        if let Some(shorthand) = shorthand {
                            return shorthand.codegen(buf);
                        }
                    }
                    RegexCharSetItem::Property { negative, value } => {
                        return value.codegen(buf, negative ^ self.negative, flavor);
                    }
                }
            }
        }

        if self.negative {
            buf.push_str("[^");
        } else {
            buf.push('[');
        }

        let mut is_first = true;
        for prop in self.set.props() {
            match prop {
                RegexCharSetItem::Shorthand(s) => s.codegen(buf),
                RegexCharSetItem::Property { negative, value } => {
                    value.codegen(buf, negative, flavor);
                }
            }
            is_first = false;
        }
        for range in self.set.ranges() {
            let (first, last) = range.as_chars();
            if first == last {
                literal::compile_char_esc_in_class(first, buf, is_first, flavor);
            } else {
                literal::compile_char_esc_in_class(first, buf, is_first, flavor);
                if range.first + 1 < range.last {
                    buf.push('-');
                }
                literal::compile_char_esc_in_class(last, buf, false, flavor);
            }
            is_first = false;
        }

        buf.push(']');
    }
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum RegexCharSetItem {
    Shorthand(RegexShorthand),
    Property { negative: bool, value: RegexProperty },
}

impl RegexCharSetItem {
    pub(crate) fn negate(self) -> Option<Self> {
        match self {
            RegexCharSetItem::Shorthand(s) => s.negate().map(RegexCharSetItem::Shorthand),
            RegexCharSetItem::Property { negative, value } => {
                Some(RegexCharSetItem::Property { negative: !negative, value })
            }
        }
    }
}

impl fmt::Debug for RegexCharSetItem {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Shorthand(s) => f.write_str(s.as_str()),
            &Self::Property { value, negative } => {
                if negative {
                    f.write_str("!")?;
                }
                f.write_str(value.prefix_as_str())?;
                f.write_str(value.as_str())
            }
        }
    }
}

1	//! Implements _character classes_. The analogue in the regex world are
2	//! [character classes](https://www.regular-expressions.info/charclass.html),
3	//! [shorthand character classes](https://www.regular-expressions.info/shorthand.html),
4	//! [non-printable characters](https://www.regular-expressions.info/nonprint.html),
5	//! [Unicode categories/scripts/blocks](https://www.regular-expressions.info/unicode.html#category),
6	//! [POSIX classes](https://www.regular-expressions.info/posixbrackets.html#class) and the
7	//! [dot](https://www.regular-expressions.info/dot.html).
8	//!
9	//! All kinds of character classes mentioned above require `[` square brackets
10	//! `]` in Pomsky. A character class can be negated by putting the keyword `not`
11	//! after the opening bracket. For example, `![.]` compiles to `\n`.
12	//!
13	//! ## Items
14	//!
15	//! A character class can contain multiple _items_, which can be
16	//!
17	//! - A __code point__, e.g. `['a']` or `[U+107]`
18	//!
19	//! - This includes [non-printable characters](https://www.regular-expressions.info/nonprint.html).\
20	//! Supported are `[n]`, `[r]`, `[t]`, `[a]`, `[e]` and `[f]`.
21	//!
22	//! - A __range of code points__. For example, `[U+10 - U+200]` matches any code
23	//! point P where `U+10 ≤ P ≤ U+200`
24	//!
25	//! - A __named character class__, which can be one of
26	//!
27	//! - a [shorthand character class](https://www.regular-expressions.info/shorthand.html).\
28	//! Supported are `[w]`, `[d]`, `[s]`, `[h]`, `[v]` and `[R]`.
29	//!
30	//! - a [POSIX class](https://www.regular-expressions.info/posixbrackets.html#class).\
31	//! Supported are `[ascii_alnum]`, `[ascii_alpha]`, `[ascii]`,
32	//! `[ascii_blank]`, `[ascii_cntrl]`, `[ascii_digit]`, `[ascii_graph]`,
33	//! `[ascii_lower]`, `[ascii_print]`, `[ascii_punct]`, ´ `[ascii_space]`,
34	//! `[ascii_upper]`, `[ascii_word]` and `[ascii_xdigit]`.\ _Note_: POSIX
35	//! classes are not Unicode aware!\ _Note_: They're converted to ranges,
36	//! e.g. `[ascii_alpha]` = `[a-zA-Z]`.
37	//!
38	//! - a [Unicode category, script or block](https://www.regular-expressions.info/unicode.html#category).\
39	//! For example: `[Letter]` compiles to `\p{Letter}`. Pomsky currently
40	//! treats any uppercase identifier except `R` as Unicode class.
41	//!
42	//! ## Compilation
43	//!
44	//! When a character class contains only a single item (e.g. `[w]`), the
45	//! character class is "flattened":
46	//!
47	//! - `['a']` = `a`
48	//! - `[w]` = `\w`
49	//! - `[Letter]` = `\p{Letter}`
50	//!
51	//! When there is more than one item or a range (e.g. `['a'-'z' '!']`), a regex
52	//! character class is created:
53	//!
54	//! - `['a'-'z' '!']` = `[a-z!]`
55	//! - `[w e Punctuation]` = `[\w\e\p{Punctuation}]`
56	//!
57	//! ### Negation
58	//!
59	//! Negation is implemented as follows:
60	//!
61	//! - Ranges and chars such as `!['a'-'z' '!' e]` are wrapped in a negative
62	//! character class, e.g. `[^a-z!\e]`.
63	//!
64	//! - The `h`, `v` and `R` shorthands are also wrapped in a negative character
65	//! class.
66	//!
67	//! - The `w`, `d` and `s` shorthands are negated by making them uppercase
68	//! (`![w]` = `\W`), except when there is more than one item in the class
69	//! (`![w '-']` = `[^\w\-]`)
70	//!
71	//! - `w`, `s`, `d` and Unicode categories/scripts/blocks can be negated
72	//! individually _within a character class_, e.g. `[s !s]` = `[\s\S]`,
73	//! `![!Latin 'a']` = `[^\P{Latin}a]`.
74	//!
75	//! When a negated character class only contains 1 item, which is also
76	//! negated, the class is removed and the negations cancel each other out:
77	//! `![!w]` = `\w`, `![!L]` = `\p{L}`.
78
79	use std::fmt;
80
81	use crate::{
82	compile::{CompileResult, CompileState},
83	diagnose::{CompileError, CompileErrorKind, Feature},
84	exprs::literal,
85	options::{CompileOptions, RegexFlavor},
86	regex::{Regex, RegexProperty, RegexShorthand},
87	unicode_set::UnicodeSet,
88	};
89
90	use pomsky_syntax::{
91	exprs::{
92	Category, CharClass, CodeBlock, GroupItem, GroupName, OtherProperties, Script,
93	ScriptExtension,
94	},
95	Span,
96	};
97
98	use super::Compile;
99
100	impl Compile for CharClass {
101	fn compile(&self, options: CompileOptions, _state: &mut CompileState<'_>) -> CompileResult {	246✔
102	// when single, a `[!w]` can be turned into `![w]`	246✔
103	let is_single = self.inner.len() == 1;	246✔
104	let mut group_negative = false;	246✔
105		246✔
106	let mut set = UnicodeSet::new();	246✔
107	for item in &self.inner {	623✔
108	match *item {	386✔
109	GroupItem::Char(c) => {	204✔
110	if !is_single {	204✔
111	validate_char_in_class(c, options.flavor, self.span)?;	140✔
112	}	64✔
113	set.add_char(c)	204✔
114	}
115	GroupItem::Range { first, last } => {	30✔
116	validate_char_in_class(first, options.flavor, self.span)?;	30✔
117	validate_char_in_class(last, options.flavor, self.span)?;	30✔
118	set.add_range(first..=last);	30✔
119	}
120	GroupItem::Named { name, negative, span } => {	152✔
121	if self.unicode_aware {	152✔
122	named_class_to_regex_unicode(	122✔
123	name,	122✔
124	negative,	122✔
125	&mut group_negative,	122✔
126	is_single,	122✔
127	options.flavor,	122✔
128	span,	122✔
129	&mut set,	122✔
130	)?;	122✔
131	} else {
132	named_class_to_regex_ascii(name, negative, options.flavor, span, &mut set)?;	30✔
133	}
134	}
135	}
136	}
137
138	// this makes it possible to use code points outside the BMP in .NET,
139	// as long as there is only one in the character set
140	if let Some(only_char) = set.try_into_char() {	237✔
141	return Ok(Regex::Literal(only_char.to_string()));	63✔
142	}	174✔
143		174✔
144	Ok(Regex::CharSet(RegexCharSet { negative: group_negative, set }))	174✔
145	}	246✔
146	}
147
148	fn validate_char_in_class(char: char, flavor: RegexFlavor, span: Span) -> Result<(), CompileError> {	200✔
149	if flavor == RegexFlavor::DotNet && char > '\u{FFFF}' {	200✔
150	Err(CompileErrorKind::Unsupported(Feature::LargeCodePointInCharClass(char), flavor)	×
151	.at(span))	×
152	} else {
153	Ok(())	200✔
154	}
155	}	200✔
156
157	pub(crate) fn check_char_class_empty(	49✔
158	char_set: &RegexCharSet,	49✔
159	span: Span,	49✔
160	) -> Result<(), CompileError> {	49✔
161	if char_set.negative {	49✔
162	if let Some((group1, group2)) = char_set.set.full_props() {	48✔
163	return Err(CompileErrorKind::EmptyClassNegated { group1, group2 }.at(span));	3✔
164	}	45✔
165	}	1✔
166	Ok(())	46✔
167	}	49✔
168
169	pub fn is_ascii_only_in_flavor(group: GroupName, flavor: RegexFlavor) -> bool {	1✔
170	match flavor {	1✔
171	RegexFlavor::JavaScript => matches!(group, GroupName::Word \| GroupName::Digit),	×
172	RegexFlavor::RE2 => matches!(group, GroupName::Word \| GroupName::Digit \| GroupName::Space),	×
173	_ => false,	1✔
174	}
175	}	1✔
176
177	fn named_class_to_regex_ascii(	30✔
178	group: GroupName,	30✔
179	negative: bool,	30✔
180	flavor: RegexFlavor,	30✔
181	span: Span,	30✔
182	set: &mut UnicodeSet,	30✔
183	) -> Result<(), CompileError> {	30✔
184	// In JS, \W and \D can be used for negation because they're ascii-only	30✔
185	// Same goes for \W, \D and \S in RE2	30✔
186	if negative && !is_ascii_only_in_flavor(group, flavor) {	30✔
187	return Err(CompileErrorKind::NegativeShorthandInAsciiMode.at(span));	1✔
188	}	29✔
189		29✔
190	match group {	29✔
191	GroupName::Word => {
192	if let RegexFlavor::JavaScript \| RegexFlavor::RE2 = flavor {	8✔
193	let s = if negative { RegexShorthand::NotWord } else { RegexShorthand::Word };	2✔
194	set.add_prop(RegexCharSetItem::Shorthand(s));	2✔
195	} else {	6✔
196	// we already checked above if negative	6✔
197	set.add_range('a'..='z');	6✔
198	set.add_range('A'..='Z');	6✔
199	set.add_range('0'..='9');	6✔
200	set.add_char('_');	6✔
201	}	6✔
202	}
203	GroupName::Digit => {
204	if let RegexFlavor::JavaScript \| RegexFlavor::RE2 = flavor {	12✔
205	let s = if negative { RegexShorthand::NotDigit } else { RegexShorthand::Digit };	2✔
206	set.add_prop(RegexCharSetItem::Shorthand(s));	2✔
207	} else {	10✔
208	// we already checked above if negative	10✔
209	set.add_range('0'..='9');	10✔
210	}	10✔
211	}
212	GroupName::Space => {
213	if let RegexFlavor::RE2 = flavor {	8✔
214	let s = if negative { RegexShorthand::NotSpace } else { RegexShorthand::Space };	1✔
215	set.add_prop(RegexCharSetItem::Shorthand(s));	1✔
216	} else {	7✔
217	set.add_char(' ');	7✔
218	set.add_range('\x09'..='\x0D'); // \t\n\v\f\r	7✔
219	}	7✔
220	}
221	GroupName::HorizSpace => set.add_char('\t'),	×
222	GroupName::VertSpace => set.add_range('\x0A'..='\x0D'),	×
223	_ => return Err(CompileErrorKind::UnicodeInAsciiMode.at(span)),	1✔
224	}
225	Ok(())	28✔
226	}	30✔
227
228	fn named_class_to_regex_unicode(	122✔
229	group: GroupName,	122✔
230	negative: bool,	122✔
231	group_negative: &mut bool,	122✔
232	is_single: bool,	122✔
233	flavor: RegexFlavor,	122✔
234	span: Span,	122✔
235	set: &mut UnicodeSet,	122✔
236	) -> Result<(), CompileError> {	122✔
237	match group {	5✔
238	GroupName::Word => {
239	if flavor == RegexFlavor::RE2 {	22✔
240	return Err(CompileErrorKind::Unsupported(Feature::ShorthandW, flavor).at(span));	×
241	} else if flavor == RegexFlavor::JavaScript {	22✔
242	if negative {	5✔
243	if is_single {	2✔
244	*group_negative ^= true;	1✔
245	} else {	1✔
246	return Err(CompileErrorKind::Unsupported(	1✔
247	Feature::NegativeShorthandW,	1✔
248	flavor,	1✔
249	)	1✔
250	.at(span));	1✔
251	}
252	}	3✔
253	set.add_prop(	4✔
254	RegexProperty::Other(OtherProperties::Alphabetic).negative_item(false),	4✔
255	);	4✔
256	set.add_prop(RegexProperty::Category(Category::Mark).negative_item(false));	4✔
257	set.add_prop(	4✔
258	RegexProperty::Category(Category::Decimal_Number).negative_item(false),	4✔
259	);	4✔
260	set.add_prop(	4✔
261	RegexProperty::Category(Category::Connector_Punctuation).negative_item(false),	4✔
262	);	4✔
263	} else {
264	let s = if negative { RegexShorthand::NotWord } else { RegexShorthand::Word };	17✔
265	set.add_prop(RegexCharSetItem::Shorthand(s));	17✔
266	}
267	}
268	GroupName::Digit => {
269	if matches!(flavor, RegexFlavor::JavaScript \| RegexFlavor::RE2) {	20✔
270	set.add_prop(	8✔
271	RegexProperty::Category(Category::Decimal_Number).negative_item(negative),	8✔
272	);	8✔
273	} else {	8✔
274	let s = if negative { RegexShorthand::NotDigit } else { RegexShorthand::Digit };	12✔
275	set.add_prop(RegexCharSetItem::Shorthand(s));	12✔
276	}
277	}
278
279	GroupName::Space => {
280	if flavor == RegexFlavor::RE2 {	16✔
281	if negative {	4✔
282	if is_single {	1✔
283	*group_negative ^= true;	1✔
284	} else {	1✔
285	return Err(CompileErrorKind::Unsupported(	×
286	Feature::NegativeShorthandS,	×
287	flavor,	×
288	)	×
289	.at(span));	×
290	}
291	}	3✔
292
293	// [ \f\n\r\t\u000b\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]
294	set.add_prop(RegexCharSetItem::Shorthand(RegexShorthand::Space));	4✔
295	set.add_char('\x0b');	4✔
296	set.add_char('\u{a0}');	4✔
297	set.add_char('\u{1680}');	4✔
298	set.add_range('\u{2000}'..='\u{200a}');	4✔
299	set.add_range('\u{2028}'..='\u{2029}');	4✔
300	set.add_char('\u{202f}');	4✔
301	set.add_char('\u{205f}');	4✔
302	set.add_char('\u{3000}');	4✔
303	set.add_char('\u{feff}');	4✔
304	} else {
305	set.add_prop(RegexCharSetItem::Shorthand(if negative {	12✔
306	RegexShorthand::NotSpace	3✔
307	} else {
308	RegexShorthand::Space	9✔
309	}))
310	}
311	}
312
313	GroupName::HorizSpace \| GroupName::VertSpace if negative => {	×
314	return Err(CompileErrorKind::NegatedHorizVertSpace.at(span));	×
315	}
316
317	GroupName::HorizSpace \| GroupName::VertSpace
318	if matches!(flavor, RegexFlavor::Pcre \| RegexFlavor::Java) =>	5✔
319	{
320	set.add_prop(RegexCharSetItem::Shorthand(if group == GroupName::HorizSpace {	6✔
321	RegexShorthand::HorizSpace	3✔
322	} else {
323	RegexShorthand::VertSpace	3✔
324	}));
325	}
326	GroupName::HorizSpace => {
327	set.add_char('\t');	2✔
328	if flavor == RegexFlavor::Python {	2✔
329	return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));	×
330	} else {	2✔
331	set.add_prop(	2✔
332	RegexProperty::Category(Category::Space_Separator).negative_item(false),	2✔
333	);	2✔
334	}	2✔
335	}
336	GroupName::VertSpace => {	2✔
337	set.add_range('\x0A'..='\x0D');	2✔
338	set.add_char('\u{85}');	2✔
339	set.add_char('\u{2028}');	2✔
340	set.add_char('\u{2029}');	2✔
341	}	2✔
342
343	_ if flavor == RegexFlavor::Python => {	54✔
344	return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));	2✔
345	}
346	GroupName::Category(c) => {	8✔
347	if let (RegexFlavor::Rust, Category::Surrogate)	8✔
348	\| (RegexFlavor::DotNet \| RegexFlavor::RE2, Category::Cased_Letter) = (flavor, c)	8✔
349	{
350	return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));	×
351	}	8✔
352	set.add_prop(RegexProperty::Category(c).negative_item(negative));	8✔
353	}
354	GroupName::Script(s, e) => {	28✔
355	if flavor == RegexFlavor::DotNet {	28✔
356	return Err(CompileErrorKind::Unsupported(Feature::UnicodeScript, flavor).at(span));	1✔
357	}	27✔
358	if let (RegexFlavor::Ruby, Script::Kawi \| Script::Nag_Mundari)	27✔
359	\| (RegexFlavor::Rust, Script::Unknown) = (flavor, s)	27✔
360	{
361	return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));	×
362	}	27✔
363
364	let set_extensions = match e {	27✔
365	ScriptExtension::Yes => match flavor {	3✔
366	RegexFlavor::Rust \| RegexFlavor::Pcre \| RegexFlavor::JavaScript => {
367	ScriptExtension::Yes	2✔
368	}
369	RegexFlavor::Java
370	\| RegexFlavor::DotNet
371	\| RegexFlavor::Ruby
372	\| RegexFlavor::Python
373	\| RegexFlavor::RE2 => {
374	return Err(CompileErrorKind::Unsupported(	1✔
375	Feature::ScriptExtensions,	1✔
376	flavor,	1✔
377	)	1✔
378	.at(span))	1✔
379	}
380	},
381	ScriptExtension::No => match flavor {	4✔
382	// PCRE is currently the only flavor when `\p{Greek}` is the same as `\p{scx=Greek}`
383	RegexFlavor::Pcre => ScriptExtension::No,	1✔
384	_ => ScriptExtension::Unspecified,	3✔
385	},
386	_ => ScriptExtension::Unspecified,	20✔
387	};
388
389	set.add_prop(RegexProperty::Script(s, set_extensions).negative_item(negative));	26✔
390	}
391	GroupName::CodeBlock(b) => match flavor {	9✔
392	RegexFlavor::DotNet \| RegexFlavor::Java \| RegexFlavor::Ruby => {
393	match (flavor, b) {	8✔
394	(RegexFlavor::Java, CodeBlock::No_Block)
395	\| (
396	// These should work since Oniguruma updated to Unicode 15.1
397	// ... but our C bindings for Oniguruma are unmaintained!
398	RegexFlavor::Ruby,
399	CodeBlock::Arabic_Extended_C
400	\| CodeBlock::CJK_Unified_Ideographs_Extension_H
401	\| CodeBlock::Cyrillic_Extended_D
402	\| CodeBlock::Devanagari_Extended_A
403	\| CodeBlock::Kaktovik_Numerals,
404	) => {
405	return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));	×
406	}
407	(RegexFlavor::DotNet, _) => {
408	let dotnet_name = b.as_str().replace("_And_", "_and_").replace('_', "");	3✔
409	if pomsky_syntax::blocks_supported_in_dotnet()	3✔
410	.binary_search(&dotnet_name.as_str())	3✔
411	.is_err()	3✔
412	{
413	return Err(	×
414	CompileErrorKind::unsupported_specific_prop_in(flavor).at(span)	×
415	);	×
416	}	3✔
417	}
418	_ => {}	5✔
419	}
420
421	set.add_prop(RegexProperty::Block(b).negative_item(negative));	8✔
422	}
423	_ => return Err(CompileErrorKind::Unsupported(Feature::UnicodeBlock, flavor).at(span)),	1✔
424	},
425	GroupName::OtherProperties(o) => {	7✔
426	use OtherProperties as OP;
427	use RegexFlavor as RF;
428
429	if let RF::JavaScript \| RF::Rust \| RF::Pcre \| RF::Ruby = flavor {	7✔
430	match (flavor, o) {	7✔
431	(RF::JavaScript, _) => {}	4✔
432	(_, OP::Changes_When_NFKC_Casefolded)
433	\| (RF::Pcre, OP::Assigned)
434	\| (RF::Ruby, OP::Bidi_Mirrored) => {
435	return Err(CompileErrorKind::unsupported_specific_prop_in(flavor).at(span));	1✔
436	}
437	_ => {}	2✔
438	}
439	set.add_prop(RegexProperty::Other(o).negative_item(negative));	6✔
440	} else {
441	return Err(CompileErrorKind::Unsupported(Feature::UnicodeProp, flavor).at(span));	×
442	}
443	}
444	}
445	Ok(())	115✔
446	}	122✔
447
448	#[cfg_attr(feature = "dbg", derive(Debug))]
449	#[derive(Default)]
450	pub(crate) struct RegexCharSet {
451	pub(crate) negative: bool,
452	pub(crate) set: UnicodeSet,
453	}
454
455	impl RegexCharSet {
456	pub(crate) fn new(items: UnicodeSet) -> Self {	244✔
457	Self { negative: false, set: items }	244✔
458	}	244✔
459
460	pub(crate) fn negate(mut self) -> Self {	49✔
461	self.negative = !self.negative;	49✔
462	self	49✔
463	}	49✔
464
465	pub(crate) fn codegen(&self, buf: &mut String, flavor: RegexFlavor) {	412✔
466	if self.set.len() == 1 {	412✔
467	if let Some(range) = self.set.ranges().next() {	319✔
468	let (first, last) = range.as_chars();	242✔
469	if first == last && !self.negative {	242✔
470	return literal::codegen_char_esc(first, buf, flavor);	75✔
471	}	167✔
472	} else if let Some(prop) = self.set.props().next() {	77✔
473	match prop {	77✔
474	RegexCharSetItem::Shorthand(s) => {	27✔
475	let shorthand = if self.negative { s.negate() } else { Some(s) };	27✔
476	if let Some(shorthand) = shorthand {	27✔
477	return shorthand.codegen(buf);	25✔
478	}	2✔
479	}
480	RegexCharSetItem::Property { negative, value } => {	50✔
481	return value.codegen(buf, negative ^ self.negative, flavor);	50✔
482	}
483	}
484	}	×
485	}	93✔
486
487	if self.negative {	262✔
488	buf.push_str("[^");	30✔
489	} else {	232✔
490	buf.push('[');	232✔
491	}	232✔
492
493	let mut is_first = true;	262✔
494	for prop in self.set.props() {	262✔
495	match prop {	76✔
496	RegexCharSetItem::Shorthand(s) => s.codegen(buf),	55✔
497	RegexCharSetItem::Property { negative, value } => {	21✔
498	value.codegen(buf, negative, flavor);	21✔
499	}	21✔
500	}
501	is_first = false;	76✔
502	}
503	for range in self.set.ranges() {	378✔
504	let (first, last) = range.as_chars();	378✔
505	if first == last {	378✔
506	literal::compile_char_esc_in_class(first, buf, is_first, flavor);	119✔
507	} else {	119✔
508	literal::compile_char_esc_in_class(first, buf, is_first, flavor);	259✔
509	if range.first + 1 < range.last {	259✔
510	buf.push('-');	207✔
511	}	207✔
512	literal::compile_char_esc_in_class(last, buf, false, flavor);	259✔
513	}
514	is_first = false;	378✔
515	}
516
517	buf.push(']');	262✔
518	}	412✔
519	}
520
521	#[derive(Clone, Copy, PartialEq, Eq)]
522	pub(crate) enum RegexCharSetItem {
523	Shorthand(RegexShorthand),
524	Property { negative: bool, value: RegexProperty },
525	}
526
527	impl RegexCharSetItem {
528	pub(crate) fn negate(self) -> Option<Self> {	47✔
529	match self {	47✔
530	RegexCharSetItem::Shorthand(s) => s.negate().map(RegexCharSetItem::Shorthand),	21✔
531	RegexCharSetItem::Property { negative, value } => {	26✔
532	Some(RegexCharSetItem::Property { negative: !negative, value })	26✔
533	}
534	}
535	}	47✔
536	}
537
538	impl fmt::Debug for RegexCharSetItem {
539	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	6✔
540	match self {	6✔
541	Self::Shorthand(s) => f.write_str(s.as_str()),	4✔
542	&Self::Property { value, negative } => {	2✔
543	if negative {	2✔
544	f.write_str("!")?;	1✔
545	}	1✔
546	f.write_str(value.prefix_as_str())?;	2✔
547	f.write_str(value.as_str())	2✔
548	}
549	}
550	}	6✔
551	}

pomsky-lang / pomsky / 12076846628

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous