14120649496

Committed 28 Mar 2025 02:25AM UTC coverage: 74.175% (-0.9%) from 75.076%

Build # 14120649496

Build Type

push

github

Committed by

vcfxb

Commit Message

nit: remove unused import

Coverage Stats

1011 of 1363 relevant lines covered (74.17%)

29.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.75

/wright/src/parser.rs

//! This parser module is responsible for turning the stream of [Token]s from the [Lexer] into a tree of [AST] nodes.
//!
//! [AST]: crate::ast
//! [Token]: crate::lexer::token::Token

use error::{ParserError, ParserErrorKind};

use super::lexer::Lexer;
use crate::{
    lexer::token::{Token, TokenTy},
    source_tracking::fragment::Fragment,
};
use std::collections::VecDeque;

mod decl;
pub mod error;
mod identifier;
mod literal;
mod path;
mod ty;
pub mod whitespace;

/// The [Parser] struct wraps a [Lexer] and adds lookahead and functions that are useful for parsing.
#[derive(Debug)]
pub struct Parser {
    lexer: Lexer,
    lookahead: VecDeque<Token>,
}

impl Parser {
    /// Construct a new parser around a given [Lexer].
    pub fn new(lexer: Lexer) -> Self {
        Parser {
            lexer,
            lookahead: VecDeque::new(),
        }
    }

    /// Get the next [Token] from this [Parser]. This may be a token that's already been peeked.
    ///
    /// Skips any non-document comments encountered via the lexer implementation.
    ///
    /// Return an error if a [Token] with [TokenTy::Unknown] is encountered.
    pub fn next_token(&mut self) -> Result<Option<Token>, ParserError> {
        let token = self
            .lookahead
            .pop_front()
            .or_else(|| self.lexer.next_token());

        // Check for unknown tokens, which should always convert to an error.
        match token {
            Some(Token {
                variant: TokenTy::Unknown,
                fragment,
            }) => Err(ParserErrorKind::EncounteredUnknownToken.at(fragment)),
            known_token_or_none => Ok(known_token_or_none),
        }
    }

    /// Advance this [Parser] by `n` [Token]s. If this [Parser] runs out of [Token]s, panic.
    ///
    /// Panics
    /// - If `n` is greater than the number of remaining tokens.
    pub fn advance(&mut self, n: usize) {
        // Add tokens to the lookahead buffer until we have enough to split off.
        while self.lookahead.len() < n {
            let token = self
                .lexer
                .next_token()
                .expect("advance: `n` <= number of remaining tokens");

            self.lookahead.push_back(token);
        }

        // Split them off.
        self.lookahead = self.lookahead.split_off(n);
    }

    /// Peek at the next token from the [Lexer] (cached in the lookahead queue if peeked before).
    pub fn peek(&mut self) -> Option<&Token> {
        if self.lookahead.is_empty() {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        self.lookahead.front()
    }

    /// Peek the [Fragment] of the next [Token].
    pub fn peek_fragment(&mut self) -> Option<&Fragment> {
        self.peek().map(|token| &token.fragment)
    }

    /// Peek the [TokenTy] of the next [Token].
    pub fn peek_variant(&mut self) -> Option<TokenTy> {
        self.peek().map(|token| token.variant)
    }

    /// Peek the [Fragment] of the next [Token] and clone it or return a clone of the
    /// remainder [Fragment] of the internal [Lexer]
    /// (which will be empty, since there wasn't a [Token] to peek).
    ///
    /// This is likely only useful for error reporting -- a clone of a potentially empty fragment is
    /// rarely ever useful otherwise.
    pub fn peek_fragment_or_rest_cloned(&mut self) -> Fragment {
        match self.peek() {
            Some(Token { fragment, .. }) => fragment.clone(),
            None => {
                let rest = self.lexer.remaining.clone();

                // Assert that we're making the right assumptions about the remaining fragment.
                // These are (unidiomatically) done using debug_assert -- perhaps that changes eventually
                // however it should be fine for now, since this can only produce logic bugs (never memory or
                // concurrency bugs).
                debug_assert!(rest.is_valid());
                debug_assert!(rest.is_empty());
                debug_assert!(rest.is_empty_at_end_of_source());

                rest
            }
        }
    }

    /// Get the [Lexer] that's wrapped.
    pub fn lexer(&self) -> &Lexer {
        &self.lexer
    }

    /// Lookahead `k` [Token]s.
    ///
    /// If `k == 0` then this is effectively peeking at the next [Token] from the wrapped [Lexer].
    pub fn lookahead(&mut self, k: usize) -> Option<&Token> {
        while self.lookahead.len() <= k {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        self.lookahead.get(k)
    }

    /// Similar to [Parser::lookahead] but instead returns a slice of `n` [Token]s, starting with the next [Token].
    ///
    /// Returns [None] if `n` is greater than the number of remaining [Token]s for this [Parser].
    pub fn lookahead_window(&mut self, n: usize) -> Option<&[Token]> {
        while self.lookahead.len() < n {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        // Use make contiguous here to get a unified/single slice.
        Some(&self.lookahead.make_contiguous()[..n])
    }

    /// Get the next [Token] from this parser if its [Token::variant] is the given `token_ty`.
    pub fn next_if_is(&mut self, token_ty: TokenTy) -> Option<Token> {
        // Peeking successfully first means that the lookahead vec will never be empty here.
        (self.peek()?.variant == token_ty)
            // SAFETY: We just peeked a token to check its variant so this unwrap is always ok.
            .then(|| unsafe { self.lookahead.pop_front().unwrap_unchecked() })
    }

    /// Peek at the next [Token]s of this [Parser] and determine if the [Token::variant]s match this
    /// sequence of [TokenTy]s.
    pub fn matches(&mut self, seq: &[TokenTy]) -> bool {
        // Use the rare let-else to ensure there are at minimum, the given number of tokens remaining.
        let Some(lookahead_window) = self.lookahead_window(seq.len()) else {
            return false;
        };

        // Use a zipped iterator to compare all the token variants.
        lookahead_window
            .iter()
            .zip(seq)
            .all(|(token, matches)| token.variant == *matches)
    }
}

1	//! This parser module is responsible for turning the stream of [Token]s from the [Lexer] into a tree of [AST] nodes.
2	//!
3	//! [AST]: crate::ast
4	//! [Token]: crate::lexer::token::Token
5
6	use error::{ParserError, ParserErrorKind};
7
8	use super::lexer::Lexer;
9	use crate::{
10	lexer::token::{Token, TokenTy},
11	source_tracking::fragment::Fragment,
12	};
13	use std::collections::VecDeque;
14
15	mod decl;
16	pub mod error;
17	mod identifier;
18	mod literal;
19	mod path;
20	mod ty;
21	pub mod whitespace;
22
23	/// The [Parser] struct wraps a [Lexer] and adds lookahead and functions that are useful for parsing.
24	#[derive(Debug)]
25	pub struct Parser {
26	lexer: Lexer,
27	lookahead: VecDeque<Token>,
28	}
29
30	impl Parser {
31	/// Construct a new parser around a given [Lexer].
32	pub fn new(lexer: Lexer) -> Self {	22✔
33	Parser {	22✔
34	lexer,	22✔
35	lookahead: VecDeque::new(),	22✔
36	}	22✔
37	}	22✔
38
39	/// Get the next [Token] from this [Parser]. This may be a token that's already been peeked.
40	///
41	/// Skips any non-document comments encountered via the lexer implementation.
42	///
43	/// Return an error if a [Token] with [TokenTy::Unknown] is encountered.
44	pub fn next_token(&mut self) -> Result<Option<Token>, ParserError> {	×
45	let token = self	×
46	.lookahead	×
47	.pop_front()	×
48	.or_else(\|\| self.lexer.next_token());	×
49
50	// Check for unknown tokens, which should always convert to an error.
51	match token {	×
52	Some(Token {
53	variant: TokenTy::Unknown,
54	fragment,	×
55	}) => Err(ParserErrorKind::EncounteredUnknownToken.at(fragment)),	×
56	known_token_or_none => Ok(known_token_or_none),	×
57	}
58	}	×
59
60	/// Advance this [Parser] by `n` [Token]s. If this [Parser] runs out of [Token]s, panic.
61	///
62	/// Panics
63	/// - If `n` is greater than the number of remaining tokens.
64	pub fn advance(&mut self, n: usize) {	12✔
65	// Add tokens to the lookahead buffer until we have enough to split off.
66	while self.lookahead.len() < n {	12✔
67	let token = self	×
68	.lexer	×
69	.next_token()	×
70	.expect("advance: `n` <= number of remaining tokens");	×
71		×
72	self.lookahead.push_back(token);	×
73	}	×
74
75	// Split them off.
76	self.lookahead = self.lookahead.split_off(n);	12✔
77	}	12✔
78
79	/// Peek at the next token from the [Lexer] (cached in the lookahead queue if peeked before).
80	pub fn peek(&mut self) -> Option<&Token> {	73✔
81	if self.lookahead.is_empty() {	73✔
82	self.lookahead.push_back(self.lexer.next_token()?);	34✔
83	}	39✔
84
85	self.lookahead.front()	71✔
86	}	73✔
87
88	/// Peek the [Fragment] of the next [Token].
89	pub fn peek_fragment(&mut self) -> Option<&Fragment> {	×
90	self.peek().map(\|token\| &token.fragment)	×
91	}	×
92
93	/// Peek the [TokenTy] of the next [Token].
94	pub fn peek_variant(&mut self) -> Option<TokenTy> {	12✔
95	self.peek().map(\|token\| token.variant)	12✔
96	}	12✔
97
98	/// Peek the [Fragment] of the next [Token] and clone it or return a clone of the
99	/// remainder [Fragment] of the internal [Lexer]
100	/// (which will be empty, since there wasn't a [Token] to peek).
101	///
102	/// This is likely only useful for error reporting -- a clone of a potentially empty fragment is
103	/// rarely ever useful otherwise.
104	pub fn peek_fragment_or_rest_cloned(&mut self) -> Fragment {	14✔
105	match self.peek() {	14✔
106	Some(Token { fragment, .. }) => fragment.clone(),	13✔
107	None => {
108	let rest = self.lexer.remaining.clone();	1✔
109		1✔
110	// Assert that we're making the right assumptions about the remaining fragment.	1✔
111	// These are (unidiomatically) done using debug_assert -- perhaps that changes eventually	1✔
112	// however it should be fine for now, since this can only produce logic bugs (never memory or	1✔
113	// concurrency bugs).	1✔
114	debug_assert!(rest.is_valid());	1✔
115	debug_assert!(rest.is_empty());	1✔
116	debug_assert!(rest.is_empty_at_end_of_source());	1✔
117
118	rest	1✔
119	}
120	}
121	}	14✔
122
123	/// Get the [Lexer] that's wrapped.
124	pub fn lexer(&self) -> &Lexer {	1✔
125	&self.lexer	1✔
126	}	1✔
127
128	/// Lookahead `k` [Token]s.
129	///
130	/// If `k == 0` then this is effectively peeking at the next [Token] from the wrapped [Lexer].
131	pub fn lookahead(&mut self, k: usize) -> Option<&Token> {	×
132	while self.lookahead.len() <= k {	×
133	self.lookahead.push_back(self.lexer.next_token()?);	×
134	}
135
136	self.lookahead.get(k)	×
137	}	×
138
139	/// Similar to [Parser::lookahead] but instead returns a slice of `n` [Token]s, starting with the next [Token].
140	///
141	/// Returns [None] if `n` is greater than the number of remaining [Token]s for this [Parser].
142	pub fn lookahead_window(&mut self, n: usize) -> Option<&[Token]> {	59✔
143	while self.lookahead.len() < n {	92✔
144	self.lookahead.push_back(self.lexer.next_token()?);	63✔
145	}
146
147	// Use make contiguous here to get a unified/single slice.
148	Some(&self.lookahead.make_contiguous()[..n])	29✔
149	}	59✔
150
151	/// Get the next [Token] from this parser if its [Token::variant] is the given `token_ty`.
152	pub fn next_if_is(&mut self, token_ty: TokenTy) -> Option<Token> {	47✔
153	// Peeking successfully first means that the lookahead vec will never be empty here.	47✔
154	(self.peek()?.variant == token_ty)	47✔
155	// SAFETY: We just peeked a token to check its variant so this unwrap is always ok.	46✔
156	.then(\|\| unsafe { self.lookahead.pop_front().unwrap_unchecked() })	46✔
157	}	47✔
158
159	/// Peek at the next [Token]s of this [Parser] and determine if the [Token::variant]s match this
160	/// sequence of [TokenTy]s.
161	pub fn matches(&mut self, seq: &[TokenTy]) -> bool {	59✔
162	// Use the rare let-else to ensure there are at minimum, the given number of tokens remaining.
163	let Some(lookahead_window) = self.lookahead_window(seq.len()) else {	59✔
164	return false;	30✔
165	};
166
167	// Use a zipped iterator to compare all the token variants.
168	lookahead_window	29✔
169	.iter()	29✔
170	.zip(seq)	29✔
171	.all(\|(token, matches)\| token.variant == *matches)	53✔
172	}	59✔
173	}

vcfxb / wright-lang / 14120649496

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous