10121430567

Committed 27 Jul 2024 06:11AM UTC coverage: 60.593% (-0.9%) from 61.486%

Build # 10121430567

Build Type

push

github

Committed by

vcfxb

Commit Message

cargo fmt

Run Details

205 of 502 branches covered (40.84%)

Branch coverage included in aggregate %.

0 of 7 new or added lines in 3 files covered. (0.0%)

25 existing lines in 2 files now uncovered.

653 of 914 relevant lines covered (71.44%)

21.12 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.83

/wright/src/parser.rs

//! This parser module is responsible for turning the stream of [Token]s from the [Lexer] into a tree of [AST] nodes.
//!
//! [AST]: crate::ast
//! [Token]: crate::lexer::token::Token

use error::{ParserError, ParserErrorKind};

use super::lexer::Lexer;
use crate::{
    lexer::token::{Token, TokenTy},
    source_tracking::fragment::Fragment,
};
use std::collections::VecDeque;

pub mod error;
mod identifier;
mod literal;
mod path;
pub mod whitespace;

/// The [Parser] struct wraps a [Lexer] and adds lookahead and functions that are useful for parsing.
#[derive(Debug)]
pub struct Parser {
    lexer: Lexer,
    lookahead: VecDeque<Token>,
}

impl Parser {
    /// Construct a new parser around a given [Lexer].
    pub fn new(lexer: Lexer) -> Self {
        Parser {
            lexer,
            lookahead: VecDeque::new(),
        }
    }

    /// Get the next [Token] from this [Parser]. This may be a token that's already been peeked.
    /// Return an error if a [Token] with [TokenTy::Unknown] is encountered.
    pub fn next_token(&mut self) -> Result<Option<Token>, ParserError> {
        let token = self
            .lookahead
            .pop_front()
            .or_else(|| self.lexer.next_token());

        // Check for unknown tokens, which should always convert to an error.
        if let Some(Token {
            variant: TokenTy::Unknown,
            fragment,
        }) = token
        {
            Err(ParserError {
                kind: ParserErrorKind::EncounteredUnknownToken,
                location: fragment,
                help: None,
            })
        } else {
            Ok(token)
        }
    }

    /// Advance this [Parser] by `n` [Token]s. If this [Parser] runs out of [Token]s, panic.
    ///
    /// Panics
    /// - If `n` is greater than the number of remaining tokens.
    pub fn advance(&mut self, n: usize) {
        // Add tokens to the lookahead buffer until we have enough to split off.
        while self.lookahead.len() < n {
            let token = self
                .lexer
                .next_token()
                .expect("advance: `n` <= number of remaining tokens");

            self.lookahead.push_back(token);
        }

        // Split them off.
        self.lookahead = self.lookahead.split_off(n);
    }

    /// Peek at the next token from the [Lexer] (cached in the lookahead queue if peeked before).
    pub fn peek(&mut self) -> Option<&Token> {
        if self.lookahead.is_empty() {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        self.lookahead.front()
    }

    /// Peek the [Fragment] of the next [Token].
    pub fn peek_fragment(&mut self) -> Option<&Fragment> {
        self.peek().map(|token| &token.fragment)
    }

    /// Get the [Lexer] that's wrapped.
    pub fn lexer(&self) -> &Lexer {
        &self.lexer
    }

    /// Lookahead `k` [Token]s.
    ///
    /// If `k == 0` then this is effectively peeking at the next [Token] from the wrapped [Lexer].
    pub fn lookahead(&mut self, k: usize) -> Option<&Token> {
        while self.lookahead.len() <= k {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        self.lookahead.get(k)
    }

    /// Similar to [Parser::lookahead] but instead returns a slice of `n` [Token]s, starting with the next [Token].
    ///
    /// Returns [None] if `n` is greater than the number of remaining [Token]s for this [Parser].
    pub fn lookahead_window(&mut self, n: usize) -> Option<&[Token]> {
        while self.lookahead.len() < n {
            self.lookahead.push_back(self.lexer.next_token()?);
        }

        // Use make contiguous here to get a unified/single slice.
        Some(&self.lookahead.make_contiguous()[..n])
    }

    /// Get the next [Token] from this parser if its [Token::variant] is the given `token_ty`.
    pub fn next_if_is(&mut self, token_ty: TokenTy) -> Option<Token> {
        // Peeking successfully first means that the lookahead vec will never be empty here.
        (self.peek()?.variant == token_ty)
            // SAFETY: We just peeked a token to check its variant so this unwrap is alway ok.
            .then(|| unsafe { self.lookahead.pop_front().unwrap_unchecked() })
    }

    /// Peek at the next [Token]s of this [Parser] and determine if the [Token::variant]s match this
    /// sequence of [TokenTy]s.
    pub fn matches(&mut self, seq: &[TokenTy]) -> bool {
        // Use the rare let-else to ensure there are at minimum, the given number of tokens remaining.
        let Some(lookahead_window) = self.lookahead_window(seq.len()) else {
            return false;
        };

        // Use a zipped iterator to compare all the token variants.
        lookahead_window
            .iter()
            .zip(seq.iter())
            .all(|(token, matches)| token.variant == *matches)
    }

    /// Peek at the next [Token], remove it if it's a [TokenTy::Whitespace].
    pub fn ignore_whitespace(&mut self) {
        self.next_if_is(TokenTy::Whitespace);
    }
}

1	//! This parser module is responsible for turning the stream of [Token]s from the [Lexer] into a tree of [AST] nodes.
2	//!
3	//! [AST]: crate::ast
4	//! [Token]: crate::lexer::token::Token
5
6	use error::{ParserError, ParserErrorKind};
7
8	use super::lexer::Lexer;
9	use crate::{
10	lexer::token::{Token, TokenTy},
11	source_tracking::fragment::Fragment,
12	};
13	use std::collections::VecDeque;
14
15	pub mod error;
16	mod identifier;
17	mod literal;
18	mod path;
19	pub mod whitespace;
20
21	/// The [Parser] struct wraps a [Lexer] and adds lookahead and functions that are useful for parsing.
22	#[derive(Debug)]	×
23	pub struct Parser {
UNCOV 24	lexer: Lexer,	×
UNCOV 25	lookahead: VecDeque<Token>,	×
26	}
27
28	impl Parser {
29	/// Construct a new parser around a given [Lexer].
30	pub fn new(lexer: Lexer) -> Self {	16✔
31	Parser {	16✔
32	lexer,
33	lookahead: VecDeque::new(),	16✔
34	}
35	}	16✔
36
37	/// Get the next [Token] from this [Parser]. This may be a token that's already been peeked.
38	/// Return an error if a [Token] with [TokenTy::Unknown] is encountered.
39	pub fn next_token(&mut self) -> Result<Option<Token>, ParserError> {	×
NEW 40	let token = self	×
41	.lookahead
42	.pop_front()
UNCOV 43	.or_else(\|\| self.lexer.next_token());	×
44
45	// Check for unknown tokens, which should always convert to an error.
NEW 46	if let Some(Token {	×
47	variant: TokenTy::Unknown,
NEW 48	fragment,	×
NEW 49	}) = token	×
50	{
UNCOV 51	Err(ParserError {	×
UNCOV 52	kind: ParserErrorKind::EncounteredUnknownToken,	×
53	location: fragment,
UNCOV 54	help: None,	×
55	})
56	} else {
UNCOV 57	Ok(token)	×
58	}
UNCOV 59	}	×
60
61	/// Advance this [Parser] by `n` [Token]s. If this [Parser] runs out of [Token]s, panic.
62	///
63	/// Panics
64	/// - If `n` is greater than the number of remaining tokens.
65	pub fn advance(&mut self, n: usize) {	4✔
66	// Add tokens to the lookahead buffer until we have enough to split off.
67	while self.lookahead.len() < n {	4!
UNCOV 68	let token = self	×
69	.lexer
70	.next_token()
71	.expect("advance: `n` <= number of remaining tokens");
72
UNCOV 73	self.lookahead.push_back(token);	×
74	}
75
76	// Split them off.
77	self.lookahead = self.lookahead.split_off(n);	4✔
78	}	4✔
79
80	/// Peek at the next token from the [Lexer] (cached in the lookahead queue if peeked before).
81	pub fn peek(&mut self) -> Option<&Token> {	30✔
82	if self.lookahead.is_empty() {	30✔
83	self.lookahead.push_back(self.lexer.next_token()?);	30✔
84	}
85
86	self.lookahead.front()	28✔
87	}	30✔
88
89	/// Peek the [Fragment] of the next [Token].
90	pub fn peek_fragment(&mut self) -> Option<&Fragment> {	10✔
91	self.peek().map(\|token\| &token.fragment)	19✔
92	}	10✔
93
94	/// Get the [Lexer] that's wrapped.
95	pub fn lexer(&self) -> &Lexer {	1✔
96	&self.lexer	1✔
97	}	1✔
98
99	/// Lookahead `k` [Token]s.
100	///
101	/// If `k == 0` then this is effectively peeking at the next [Token] from the wrapped [Lexer].
UNCOV 102	pub fn lookahead(&mut self, k: usize) -> Option<&Token> {	×
UNCOV 103	while self.lookahead.len() <= k {	×
UNCOV 104	self.lookahead.push_back(self.lexer.next_token()?);	×
105	}
106
UNCOV 107	self.lookahead.get(k)	×
UNCOV 108	}	×
109
110	/// Similar to [Parser::lookahead] but instead returns a slice of `n` [Token]s, starting with the next [Token].
111	///
112	/// Returns [None] if `n` is greater than the number of remaining [Token]s for this [Parser].
113	pub fn lookahead_window(&mut self, n: usize) -> Option<&[Token]> {	26✔
114	while self.lookahead.len() < n {	38✔
115	self.lookahead.push_back(self.lexer.next_token()?);	59✔
116	}
117
118	// Use make contiguous here to get a unified/single slice.
119	Some(&self.lookahead.make_contiguous()[..n])	5✔
120	}	26✔
121
122	/// Get the next [Token] from this parser if its [Token::variant] is the given `token_ty`.
123	pub fn next_if_is(&mut self, token_ty: TokenTy) -> Option<Token> {	20✔
124	// Peeking successfully first means that the lookahead vec will never be empty here.
125	(self.peek()?.variant == token_ty)	20✔
126	// SAFETY: We just peeked a token to check its variant so this unwrap is alway ok.
127	.then(\|\| unsafe { self.lookahead.pop_front().unwrap_unchecked() })	10✔
128	}	20✔
129
130	/// Peek at the next [Token]s of this [Parser] and determine if the [Token::variant]s match this
131	/// sequence of [TokenTy]s.
132	pub fn matches(&mut self, seq: &[TokenTy]) -> bool {	26✔
133	// Use the rare let-else to ensure there are at minimum, the given number of tokens remaining.
134	let Some(lookahead_window) = self.lookahead_window(seq.len()) else {	26✔
135	return false;	21✔
136	};
137
138	// Use a zipped iterator to compare all the token variants.
139	lookahead_window	10✔
140	.iter()
141	.zip(seq.iter())	5✔
142	.all(\|(token, matches)\| token.variant == *matches)	13✔
143	}	26✔
144
145	/// Peek at the next [Token], remove it if it's a [TokenTy::Whitespace].
UNCOV 146	pub fn ignore_whitespace(&mut self) {	×
UNCOV 147	self.next_if_is(TokenTy::Whitespace);	×
UNCOV 148	}	×
149	}

vcfxb / wright-lang / 10121430567

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous