9142134100

Committed 18 May 2024 07:23PM UTC coverage: 82.949% (-1.3%) from 84.258%

Build # 9142134100

Build Type

push

github

Committed by

Aloso

Commit Message

fix e2e tests

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

271 existing lines in 24 files now uncovered.

4242 of 5114 relevant lines covered (82.95%)

420176.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.62

/pomsky-syntax/src/parse/parser.rs

use std::str::FromStr;

use crate::{
    diagnose::{
        LexErrorMsg, NumberError, ParseDiagnostic, ParseError, ParseErrorKind as PEK, ParseWarning,
    },
    exprs::*,
    lexer::{tokenize, Token},
    Span,
};

/// Parses a source string as a pomsky expression.
///
/// The `recursion` argument determines how much nesting is allowed in the
/// expression. Note that **pomsky will overflow the stack** when parsing an
/// expression with too much nesting, so the `recursion` argument should be low
/// enough to prevent that. The recommended default is 256.
pub fn parse(source: &str, recursion: u32) -> (Option<Rule>, Vec<ParseDiagnostic>) {
    if source.len() > u32::MAX as usize {
        let error = PEK::LexErrorWithMessage(LexErrorMsg::FileTooBig);
        return (None, vec![error.at(Span::empty()).into()]);
    }

    let tokens = tokenize(source);

    let mut errors = Vec::new();
    for &(t, span) in &tokens {
        match t {
            Token::Error => errors.push((span, None)),
            Token::ErrorMsg(m) => errors.push((span, Some(m))),
            _ => {}
        }
    }

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(span, msg)| {
                msg.map_or(PEK::UnknownToken, PEK::LexErrorWithMessage).at(span).into()
            })
            .collect::<Vec<_>>();

        return (None, errors);
    }

    let mut parser = Parser {
        source,
        tokens: tokens.into_boxed_slice(),
        offset: 0,
        warnings: Vec::new(),
        recursion,
        is_lazy: false,
        is_unicode_aware: true,
    };

    let rule = match parser.parse_modified() {
        Ok(rule) => rule,
        Err(err) => {
            let mut diagnostics = vec![err.into()];
            diagnostics.extend(parser.warnings);
            return (None, diagnostics);
        }
    };
    if parser.is_empty() {
        (Some(rule), parser.warnings)
    } else {
        let mut diagnostics = vec![PEK::LeftoverTokens.at(parser.span()).into()];
        diagnostics.extend(parser.warnings);
        (None, diagnostics)
    }
}

type PResult<T> = Result<T, ParseError>;

pub(super) struct Parser<'i> {
    source: &'i str,
    tokens: Box<[(Token, Span)]>,
    offset: usize,
    warnings: Vec<ParseDiagnostic>,
    recursion: u32,
    pub(super) is_lazy: bool,
    pub(super) is_unicode_aware: bool,
}

// Utilities
impl<'i> Parser<'i> {
    pub(super) fn is_empty(&self) -> bool {
        self.tokens.len() == self.offset
    }

    pub(super) fn source_at(&self, span: Span) -> &'i str {
        &self.source[span.range_unchecked()]
    }

    pub(super) fn peek(&self) -> Option<(Token, &'i str)> {
        self.tokens.get(self.offset).map(|&(t, span)| (t, self.source_at(span)))
    }

    pub(super) fn peek_pair(&self) -> Option<(Token, Span)> {
        self.tokens.get(self.offset).copied()
    }

    /// Returns the span of the next token
    pub(super) fn span(&self) -> Span {
        self.tokens
            .get(self.offset)
            .map_or_else(|| Span::new(self.source.len(), self.source.len()), |&(_, s)| s)
    }

    /// Returns the span of the previously consumed token
    pub(super) fn last_span(&self) -> Span {
        self.tokens[self.offset - 1].1
    }

    pub(super) fn advance(&mut self) {
        self.offset += 1;
    }

    pub(super) fn recursion_start(&mut self) -> PResult<()> {
        self.recursion =
            self.recursion.checked_sub(1).ok_or_else(|| PEK::RecursionLimit.at(self.span()))?;
        Ok(())
    }

    pub(super) fn recursion_end(&mut self) {
        self.recursion += 1;
    }

    pub(super) fn add_warning(&mut self, warning: ParseWarning) {
        self.warnings.push(warning.into());
    }

    pub(super) fn is(&mut self, token: Token) -> bool {
        matches!(self.peek_pair(), Some((t, _)) if t == token)
    }

    pub(super) fn consume(&mut self, token: Token) -> bool {
        match self.peek_pair() {
            Some((t, _)) if t == token => {
                self.offset += 1;
                true
            }
            _ => false,
        }
    }

    pub(super) fn consume_as(&mut self, token: Token) -> Option<&'i str> {
        match self.peek_pair() {
            Some((t, span)) if t == token => {
                self.offset += 1;
                Some(self.source_at(span))
            }
            _ => None,
        }
    }

    pub(super) fn consume_reserved(&mut self, reserved: &str) -> bool {
        match self.peek_pair() {
            Some((Token::ReservedName, s)) if self.source_at(s) == reserved => {
                self.offset += 1;
                true
            }
            _ => false,
        }
    }

    pub(super) fn consume_contextual_keyword(&mut self, keyword: &str) -> bool {
        match self.peek_pair() {
            Some((Token::Identifier, s)) if self.source_at(s) == keyword => {
                self.offset += 1;
                true
            }
            _ => false,
        }
    }

    pub(super) fn consume_number<T: FromStr + PartialOrd>(&mut self, max: T) -> PResult<Option<T>> {
        match self.peek_pair() {
            Some((Token::Number, span)) => {
                let n = str::parse(self.source_at(span))
                    .ok()
                    .and_then(|n| if n > max { None } else { Some(n) })
                    .ok_or_else(|| PEK::Number(NumberError::TooLarge).at(span))?;
                self.offset += 1;
                Ok(Some(n))
            }
            _ => Ok(None),
        }
    }

    pub(super) fn expect(&mut self, token: Token) -> PResult<()> {
        match self.peek_pair() {
            Some((t, _)) if t == token => {
                self.offset += 1;
                Ok(())
            }
            _ => Err(PEK::ExpectedToken(token).at(self.span())),
        }
    }

    pub(super) fn expect_as(&mut self, token: Token) -> PResult<&'i str> {
        match self.peek_pair() {
            Some((t, span)) if t == token => {
                self.offset += 1;
                Ok(self.source_at(span))
            }
            _ => Err(PEK::ExpectedToken(token).at(self.span())),
        }
    }

    pub(super) fn expect_number<T: FromStr>(&mut self) -> PResult<T> {
        match self.peek_pair() {
            Some((Token::Number, span)) => {
                let n = str::parse(self.source_at(span))
                    .map_err(|_| PEK::Number(NumberError::TooLarge).at(span))?;
                self.offset += 1;
                Ok(n)
            }
            _ => Err(PEK::ExpectedToken(Token::Number).at(self.span())),
        }
    }
}

1	use std::str::FromStr;
2
3	use crate::{
4	diagnose::{
5	LexErrorMsg, NumberError, ParseDiagnostic, ParseError, ParseErrorKind as PEK, ParseWarning,
6	},
7	exprs::*,
8	lexer::{tokenize, Token},
9	Span,
10	};
11
12	/// Parses a source string as a pomsky expression.
13	///
14	/// The `recursion` argument determines how much nesting is allowed in the
15	/// expression. Note that pomsky will overflow the stack when parsing an
16	/// expression with too much nesting, so the `recursion` argument should be low
17	/// enough to prevent that. The recommended default is 256.
18	pub fn parse(source: &str, recursion: u32) -> (Option<Rule>, Vec<ParseDiagnostic>) {	365✔
19	if source.len() > u32::MAX as usize {	365✔
UNCOV 20	let error = PEK::LexErrorWithMessage(LexErrorMsg::FileTooBig);	×
UNCOV 21	return (None, vec![error.at(Span::empty()).into()]);	×
22	}	365✔
23		365✔
24	let tokens = tokenize(source);	365✔
25		365✔
26	let mut errors = Vec::new();	365✔
27	for &(t, span) in &tokens {	4,312✔
28	match t {	3,947✔
29	Token::Error => errors.push((span, None)),	8✔
30	Token::ErrorMsg(m) => errors.push((span, Some(m))),	48✔
31	_ => {}	3,891✔
32	}
33	}
34
35	if !errors.is_empty() {	365✔
36	let errors = errors	47✔
37	.into_iter()	47✔
38	.map(\|(span, msg)\| {	56✔
39	msg.map_or(PEK::UnknownToken, PEK::LexErrorWithMessage).at(span).into()	56✔
40	})	56✔
41	.collect::<Vec<_>>();	47✔
42		47✔
43	return (None, errors);	47✔
44	}	318✔
45		318✔
46	let mut parser = Parser {	318✔
47	source,	318✔
48	tokens: tokens.into_boxed_slice(),	318✔
49	offset: 0,	318✔
50	warnings: Vec::new(),	318✔
51	recursion,	318✔
52	is_lazy: false,	318✔
53	is_unicode_aware: true,	318✔
54	};	318✔
55
56	let rule = match parser.parse_modified() {	318✔
57	Ok(rule) => rule,	256✔
58	Err(err) => {	62✔
59	let mut diagnostics = vec![err.into()];	62✔
60	diagnostics.extend(parser.warnings);	62✔
61	return (None, diagnostics);	62✔
62	}
63	};
64	if parser.is_empty() {	256✔
65	(Some(rule), parser.warnings)	253✔
66	} else {
67	let mut diagnostics = vec![PEK::LeftoverTokens.at(parser.span()).into()];	3✔
68	diagnostics.extend(parser.warnings);	3✔
69	(None, diagnostics)	3✔
70	}
71	}	365✔
72
73	type PResult<T> = Result<T, ParseError>;
74
75	pub(super) struct Parser<'i> {
76	source: &'i str,
77	tokens: Box<[(Token, Span)]>,
78	offset: usize,
79	warnings: Vec<ParseDiagnostic>,
80	recursion: u32,
81	pub(super) is_lazy: bool,
82	pub(super) is_unicode_aware: bool,
83	}
84
85	// Utilities
86	impl<'i> Parser<'i> {
87	pub(super) fn is_empty(&self) -> bool {	256✔
88	self.tokens.len() == self.offset	256✔
89	}	256✔
90
91	pub(super) fn source_at(&self, span: Span) -> &'i str {	2,086✔
92	&self.source[span.range_unchecked()]	2,086✔
93	}	2,086✔
94
95	pub(super) fn peek(&self) -> Option<(Token, &'i str)> {	438✔
96	self.tokens.get(self.offset).map(\|&(t, span)\| (t, self.source_at(span)))	438✔
97	}	438✔
98
99	pub(super) fn peek_pair(&self) -> Option<(Token, Span)> {	38,386✔
100	self.tokens.get(self.offset).copied()	38,386✔
101	}	38,386✔
102
103	/// Returns the span of the next token
104	pub(super) fn span(&self) -> Span {	7,832✔
105	self.tokens	7,832✔
106	.get(self.offset)	7,832✔
107	.map_or_else(\|\| Span::new(self.source.len(), self.source.len()), \|&(_, s)\| s)	7,832✔
108	}	7,832✔
109
110	/// Returns the span of the previously consumed token
111	pub(super) fn last_span(&self) -> Span {	2,172✔
112	self.tokens[self.offset - 1].1	2,172✔
113	}	2,172✔
114
115	pub(super) fn advance(&mut self) {	61✔
116	self.offset += 1;	61✔
117	}	61✔
118
119	pub(super) fn recursion_start(&mut self) -> PResult<()> {	1,277✔
120	self.recursion =	1,277✔
121	self.recursion.checked_sub(1).ok_or_else(\|\| PEK::RecursionLimit.at(self.span()))?;	1,277✔
122	Ok(())	1,275✔
123	}	1,277✔
124
125	pub(super) fn recursion_end(&mut self) {	708✔
126	self.recursion += 1;	708✔
127	}	708✔
128
129	pub(super) fn add_warning(&mut self, warning: ParseWarning) {	2✔
130	self.warnings.push(warning.into());	2✔
131	}	2✔
132
133	pub(super) fn is(&mut self, token: Token) -> bool {	2✔
134	matches!(self.peek_pair(), Some((t, _)) if t == token)	2✔
135	}	2✔
136
137	pub(super) fn consume(&mut self, token: Token) -> bool {	24,947✔
138	match self.peek_pair() {	24,947✔
139	Some((t, _)) if t == token => {	19,946✔
140	self.offset += 1;	1,316✔
141	true	1,316✔
142	}
143	_ => false,	23,631✔
144	}
145	}	24,947✔
146
147	pub(super) fn consume_as(&mut self, token: Token) -> Option<&'i str> {	4,412✔
148	match self.peek_pair() {	4,412✔
149	Some((t, span)) if t == token => {	3,602✔
150	self.offset += 1;	686✔
151	Some(self.source_at(span))	686✔
152	}
153	_ => None,	3,726✔
154	}
155	}	4,412✔
156
157	pub(super) fn consume_reserved(&mut self, reserved: &str) -> bool {	7,684✔
158	match self.peek_pair() {	7,684✔
159	Some((Token::ReservedName, s)) if self.source_at(s) == reserved => {	516✔
160	self.offset += 1;	151✔
161	true	151✔
162	}
163	_ => false,	7,533✔
164	}
165	}	7,684✔
166
167	pub(super) fn consume_contextual_keyword(&mut self, keyword: &str) -> bool {	131✔
168	match self.peek_pair() {	131✔
169	Some((Token::Identifier, s)) if self.source_at(s) == keyword => {	69✔
170	self.offset += 1;	55✔
171	true	55✔
172	}
173	_ => false,	76✔
174	}
175	}	131✔
176
177	pub(super) fn consume_number<T: FromStr + PartialOrd>(&mut self, max: T) -> PResult<Option<T>> {	139✔
178	match self.peek_pair() {	139✔
179	Some((Token::Number, span)) => {	85✔
180	let n = str::parse(self.source_at(span))	85✔
181	.ok()	85✔
182	.and_then(\|n\| if n > max { None } else { Some(n) })	85✔
183	.ok_or_else(\|\| PEK::Number(NumberError::TooLarge).at(span))?;	85✔
184	self.offset += 1;	84✔
185	Ok(Some(n))	84✔
186	}
187	_ => Ok(None),	54✔
188	}
189	}	139✔
190
191	pub(super) fn expect(&mut self, token: Token) -> PResult<()> {	720✔
192	match self.peek_pair() {	720✔
193	Some((t, _)) if t == token => {	715✔
194	self.offset += 1;	709✔
195	Ok(())	709✔
196	}
197	_ => Err(PEK::ExpectedToken(token).at(self.span())),	11✔
198	}
199	}	720✔
200
201	pub(super) fn expect_as(&mut self, token: Token) -> PResult<&'i str> {	135✔
202	match self.peek_pair() {	135✔
203	Some((t, span)) if t == token => {	134✔
204	self.offset += 1;	132✔
205	Ok(self.source_at(span))	132✔
206	}
207	_ => Err(PEK::ExpectedToken(token).at(self.span())),	3✔
208	}
209	}	135✔
210
211	pub(super) fn expect_number<T: FromStr>(&mut self) -> PResult<T> {	6✔
212	match self.peek_pair() {	6✔
213	Some((Token::Number, span)) => {	5✔
214	let n = str::parse(self.source_at(span))	5✔
215	.map_err(\|_\| PEK::Number(NumberError::TooLarge).at(span))?;	5✔
216	self.offset += 1;	5✔
217	Ok(n)	5✔
218	}
219	_ => Err(PEK::ExpectedToken(Token::Number).at(self.span())),	1✔
220	}
221	}	6✔
222	}

pomsky-lang / pomsky / 9142134100

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous