13615020054

Committed 02 Mar 2025 11:34AM UTC coverage: 47.089% (+0.03%) from 47.062%

Build # 13615020054

Build Type

Pull #310

github

Committed by

web-flow

Commit Message

Merge f80ed61d2 into f1a605a0f

Pull Request Pull Request #310: support escaped identifiers

Run Details

93 of 138 new or added lines in 4 files covered. (67.39%)

4 existing lines in 2 files now uncovered.

12737 of 27049 relevant lines covered (47.09%)

422185.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.76

/crates/steel-parser/src/tokens.rs

use crate::lexer;
use crate::parser::SourceId;
use crate::span::Span;
use core::ops;
use num::{BigInt, Rational32, Signed};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::fmt::{self, Display};
use std::str::FromStr;
use std::sync::Arc;
use TokenType::*;

#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Paren {
    Round,
    Square,
    Curly,
}

#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum ParenMod {
    Vector,
    Bytes,
}

impl ParenMod {
    pub(crate) fn as_str(&self) -> &'static str {
        match self {
            ParenMod::Vector => "#",
            ParenMod::Bytes => "#u8",
        }
    }
}

impl Display for ParenMod {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.as_str().fmt(f)
    }
}

impl Paren {
    pub fn open(&self) -> char {
        match self {
            Paren::Round => '(',
            Paren::Square => '[',
            Paren::Curly => '{',
        }
    }

    pub fn close(&self) -> char {
        match self {
            Paren::Round => ')',
            Paren::Square => ']',
            Paren::Curly => '}',
        }
    }
}

// TODO the character parsing is not quite right
// need to make sure that we can handle cases like "#\SPACE" or "#\a" but not "#\applesauce"
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum TokenType<S> {
    OpenParen(Paren, Option<ParenMod>),
    CloseParen(Paren),
    QuoteTick,
    QuasiQuote,
    Unquote,
    UnquoteSplice,
    QuoteSyntax,
    QuasiQuoteSyntax,
    UnquoteSyntax,
    UnquoteSpliceSyntax,
    If,
    Define,
    Let,
    TestLet,
    Return,
    Begin,
    Lambda,
    Quote,
    SyntaxRules,
    DefineSyntax,
    Ellipses,
    Set,
    Require,
    CharacterLiteral(char),
    DatumComment,
    Comment,
    BooleanLiteral(bool),
    Identifier(S),
    Keyword(S),
    Number(Box<NumberLiteral>),
    // TODO: Consider using Arc<String> here instead to save
    // on copies
    StringLiteral(Arc<String>),
    Dot,
    Error,
}

impl<T> TokenType<T> {
    pub fn identifier_mut(&mut self) -> Option<&mut T> {
        if let Self::Identifier(i) = self {
            Some(i)
        } else {
            None
        }
    }
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum NumberLiteral {
    Real(RealLiteral),
    Complex(RealLiteral, RealLiteral),
}

impl Display for NumberLiteral {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            NumberLiteral::Real(r) => r.fmt(f),
            NumberLiteral::Complex(re, im) => {
                if im.is_negative() {
                    write!(f, "{re}{im}i")
                } else {
                    write!(f, "{re}+{im}i")
                }
            }
        }
    }
}

impl<S> From<NumberLiteral> for TokenType<S> {
    fn from(n: NumberLiteral) -> Self {
        TokenType::Number(Box::new(n))
    }
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum RealLiteral {
    Int(IntLiteral),
    Rational(IntLiteral, IntLiteral),
    Float(f64),
}

impl RealLiteral {
    fn is_negative(&self) -> bool {
        match self {
            RealLiteral::Int(i) => i.is_negative(),
            RealLiteral::Rational(n, _) => n.is_negative(),
            RealLiteral::Float(f) => f.is_sign_negative(),
        }
    }
}

impl From<RealLiteral> for NumberLiteral {
    fn from(value: RealLiteral) -> Self {
        NumberLiteral::Real(value).into()
    }
}

impl<S> From<RealLiteral> for TokenType<S> {
    fn from(value: RealLiteral) -> Self {
        NumberLiteral::Real(value).into()
    }
}

impl From<f64> for RealLiteral {
    fn from(value: f64) -> RealLiteral {
        RealLiteral::Float(value)
    }
}

impl From<isize> for RealLiteral {
    fn from(value: isize) -> RealLiteral {
        RealLiteral::Int(IntLiteral::Small(value))
    }
}

impl From<Rational32> for RealLiteral {
    fn from(value: Rational32) -> RealLiteral {
        RealLiteral::Rational(
            (*value.numer() as isize).into(),
            (*value.denom() as isize).into(),
        )
    }
}

impl Display for RealLiteral {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            RealLiteral::Int(i) => i.fmt(f),
            RealLiteral::Rational(n, d) => write!(f, "{n}/{d}"),
            RealLiteral::Float(x) => {
                if x.is_nan() {
                    write!(f, "{}", lexer::NAN)
                } else if x.is_infinite() && x.is_sign_negative() {
                    write!(f, "{}", lexer::NEG_INFINITY)
                } else if x.is_infinite() {
                    write!(f, "{}", lexer::INFINITY)
                } else {
                    x.fmt(f)
                }
            }
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum IntLiteral {
    Small(isize),
    Big(Box<BigInt>),
}

impl IntLiteral {
    fn is_negative(&self) -> bool {
        match self {
            IntLiteral::Small(i) => i.is_negative(),
            IntLiteral::Big(i) => i.is_negative(),
        }
    }
}

impl FromStr for IntLiteral {
    type Err = <num::BigInt as FromStr>::Err;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        s.parse::<isize>().map(IntLiteral::Small).or_else(|_| {
            s.parse::<num::BigInt>()
                .map(|b| IntLiteral::Big(Box::new(b)))
        })
    }
}

impl Display for IntLiteral {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Small(s) => write!(f, "{s}"),
            Self::Big(b) => write!(f, "{b}"),
        }
    }
}

impl<S> From<IntLiteral> for TokenType<S> {
    fn from(value: IntLiteral) -> Self {
        RealLiteral::Int(value).into()
    }
}

impl From<IntLiteral> for RealLiteral {
    fn from(value: IntLiteral) -> Self {
        RealLiteral::Int(value)
    }
}

impl From<IntLiteral> for BigInt {
    fn from(v: IntLiteral) -> BigInt {
        match v {
            IntLiteral::Small(x) => x.into(),
            IntLiteral::Big(x) => *x,
        }
    }
}

impl From<isize> for IntLiteral {
    fn from(value: isize) -> Self {
        IntLiteral::Small(value)
    }
}

impl From<BigInt> for IntLiteral {
    fn from(value: BigInt) -> Self {
        IntLiteral::Big(Box::new(value))
    }
}

impl<'a> TokenType<Cow<'a, str>> {
    pub fn open_span(mut span: Span, paren_mod: Option<ParenMod>) -> Span {
        let offset = match paren_mod {
            Some(ParenMod::Vector) => 1,
            Some(ParenMod::Bytes) => 3,
            None => 0,
        };

        span.start += offset;

        span
    }

    pub fn to_owned<T: From<Cow<'a, str>>>(self) -> TokenType<T> {
        match self {
            TokenType::Identifier(i) => TokenType::Identifier(i.into()),
            TokenType::Keyword(i) => TokenType::Keyword(i.into()),
            OpenParen(p, m) => OpenParen(p, m),
            CloseParen(p) => CloseParen(p),
            CharacterLiteral(x) => CharacterLiteral(x),
            BooleanLiteral(x) => BooleanLiteral(x),
            Number(x) => Number(x),
            StringLiteral(x) => StringLiteral(x),
            QuoteTick => QuoteTick,
            Unquote => Unquote,
            QuasiQuote => QuasiQuote,
            UnquoteSplice => UnquoteSplice,
            Error => Error,
            Comment => Comment,
            DatumComment => DatumComment,
            If => If,
            Define => Define,
            Let => Let,
            TestLet => TestLet,
            Return => Return,
            Begin => Begin,
            Lambda => Lambda,
            Quote => Quote,
            DefineSyntax => DefineSyntax,
            SyntaxRules => SyntaxRules,
            Ellipses => Ellipses,
            Set => Set,
            Require => Require,
            QuasiQuoteSyntax => QuasiQuoteSyntax,
            UnquoteSyntax => UnquoteSyntax,
            QuoteSyntax => QuoteSyntax,
            UnquoteSpliceSyntax => UnquoteSpliceSyntax,
            Dot => Dot,
        }
    }

    pub fn map<T>(self, mut func: impl FnMut(Cow<'a, str>) -> T) -> TokenType<T> {
        match self {
            TokenType::Identifier(i) => TokenType::Identifier(func(i)),
            TokenType::Keyword(i) => TokenType::Keyword(func(i)),
            OpenParen(p, m) => OpenParen(p, m),
            CloseParen(p) => CloseParen(p),
            CharacterLiteral(x) => CharacterLiteral(x),
            BooleanLiteral(x) => BooleanLiteral(x),
            Number(x) => Number(x),
            StringLiteral(x) => StringLiteral(x),
            QuoteTick => QuoteTick,
            Unquote => Unquote,
            QuasiQuote => QuasiQuote,
            UnquoteSplice => UnquoteSplice,
            Error => Error,
            Comment => Comment,
            DatumComment => DatumComment,
            If => If,
            Define => Define,
            Let => Let,
            TestLet => TestLet,
            Return => Return,
            Begin => Begin,
            Lambda => Lambda,
            Quote => Quote,
            DefineSyntax => DefineSyntax,
            SyntaxRules => SyntaxRules,
            Ellipses => Ellipses,
            Set => Set,
            Require => Require,
            QuasiQuoteSyntax => QuasiQuoteSyntax,
            UnquoteSyntax => UnquoteSyntax,
            QuoteSyntax => QuoteSyntax,
            UnquoteSpliceSyntax => UnquoteSpliceSyntax,
            Dot => Dot,
        }
    }
}

fn character_special_display(c: char, f: &mut fmt::Formatter) -> fmt::Result {
    match c {
        ' ' => write!(f, "#\\SPACE"),
        '\t' => write!(f, "#\\TAB"),
        '\n' => write!(f, "#\\NEWLINE"),
        '\r' => write!(f, "#\\RETURN"),
        c if c.is_control() || c.is_whitespace() => {
            write!(f, "#\\{}", c.escape_unicode())
        }
        // '\"' => write!(f, "#\\DOUBLE-QUOTE"),
        // '\'' => write!(f, "#\\QUOTE"),
        // '\\' => write!(f, "#\\BACKSLASH"),
        _ => write!(f, "#\\{c}"),
    }
}

impl<T: Display> fmt::Display for TokenType<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            OpenParen(p, m) => {
                if let Some(m) = m {
                    m.fmt(f)?;
                }

                write!(f, "{}", p.open())
            }
            CloseParen(p) => write!(f, "{}", p.close()),
            CharacterLiteral(x) => character_special_display(*x, f),
            BooleanLiteral(x) => write!(f, "#{x}"),
            Identifier(x) => write!(f, "{x}"),
            Number(x) => write!(f, "{x}"),
            StringLiteral(x) => write!(f, "\"{x}\""),
            Keyword(x) => write!(f, "{x}"),
            QuoteTick => write!(f, "'"),
            Unquote => write!(f, ","),
            QuasiQuote => write!(f, "`"),
            UnquoteSplice => write!(f, ",@"),
            QuoteSyntax => write!(f, "#'"),
            QuasiQuoteSyntax => write!(f, "#`"),
            UnquoteSyntax => write!(f, "#,"),
            UnquoteSpliceSyntax => write!(f, "#,@"),
            Error => write!(f, "error"),
            DatumComment => write!(f, "#;"),
            Comment => write!(f, ""),
            If => write!(f, "if"),
            Define => write!(f, "define"),
            Let => write!(f, "let"),
            TestLet => write!(f, "%plain-let"),
            Return => write!(f, "return!"),
            Begin => write!(f, "begin"),
            Lambda => write!(f, "lambda"),
            Quote => write!(f, "quote"),
            DefineSyntax => write!(f, "define-syntax"),
            SyntaxRules => write!(f, "syntax-rules"),
            Ellipses => write!(f, "..."),
            Set => write!(f, "set!"),
            Require => write!(f, "require"),
            Dot => write!(f, "."),
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct Token<'a, T> {
    pub ty: TokenType<T>,
    pub source: &'a str,
    pub span: Span,
}

impl<'a, T> Token<'a, T> {
    pub const fn new(
        ty: TokenType<T>,
        source: &'a str,
        range: ops::Range<usize>,
        source_id: Option<SourceId>,
    ) -> Self {
        Self {
            ty,
            source,
            span: Span::new(range.start, range.end, source_id),
        }
    }

    pub fn typ(&self) -> &TokenType<T> {
        &self.ty
    }

    pub const fn span(&self) -> Span {
        self.span
    }

    pub const fn range(&self) -> ops::Range<usize> {
        self.span.start()..self.span.end()
    }

    pub const fn source(&self) -> &'a str {
        self.source
    }
}

impl<T> From<Token<'_, T>> for Span {
    fn from(token: Token<'_, T>) -> Self {
        token.span()
    }
}

impl<T> From<&Token<'_, T>> for Span {
    fn from(token: &Token<'_, T>) -> Self {
        token.span()
    }
}

impl<T> From<Token<'_, T>> for ops::Range<usize> {
    fn from(token: Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> From<&Token<'_, T>> for ops::Range<usize> {
    fn from(token: &Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> From<Token<'_, T>> for (usize, usize) {
    fn from(token: Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> From<&Token<'_, T>> for (usize, usize) {
    fn from(token: &Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> From<Token<'_, T>> for [usize; 2] {
    fn from(token: Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> From<&Token<'_, T>> for [usize; 2] {
    fn from(token: &Token<'_, T>) -> Self {
        token.span().into()
    }
}

impl<T> Display for Token<'_, T> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{} @ {:?}", self.source, self.span)
    }
}

1	use crate::lexer;
2	use crate::parser::SourceId;
3	use crate::span::Span;
4	use core::ops;
5	use num::{BigInt, Rational32, Signed};
6	use serde::{Deserialize, Serialize};
7	use std::borrow::Cow;
8	use std::fmt::{self, Display};
9	use std::str::FromStr;
10	use std::sync::Arc;
11	use TokenType::*;
12
13	#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
14	pub enum Paren {
15	Round,
16	Square,
17	Curly,
18	}
19
20	#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
21	pub enum ParenMod {
22	Vector,
23	Bytes,
24	}
25
26	impl ParenMod {
27	pub(crate) fn as_str(&self) -> &'static str {	×
28	match self {	×
29	ParenMod::Vector => "#",	×
30	ParenMod::Bytes => "#u8",	×
31	}
32	}
33	}
34
35	impl Display for ParenMod {
36	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	1✔
37	self.as_str().fmt(f)	1✔
38	}
39	}
40
41	impl Paren {
42	pub fn open(&self) -> char {	×
43	match self {	×
44	Paren::Round => '(',	×
45	Paren::Square => '[',	×
46	Paren::Curly => '{',	×
47	}
48	}
49
50	pub fn close(&self) -> char {	×
51	match self {	×
52	Paren::Round => ')',	×
53	Paren::Square => ']',	×
54	Paren::Curly => '}',	×
55	}
56	}
57	}
58
59	// TODO the character parsing is not quite right
60	// need to make sure that we can handle cases like "#\SPACE" or "#\a" but not "#\applesauce"
61	#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
62	pub enum TokenType<S> {
63	OpenParen(Paren, Option<ParenMod>),
64	CloseParen(Paren),
65	QuoteTick,
66	QuasiQuote,
67	Unquote,
68	UnquoteSplice,
69	QuoteSyntax,
70	QuasiQuoteSyntax,
71	UnquoteSyntax,
72	UnquoteSpliceSyntax,
73	If,
74	Define,
75	Let,
76	TestLet,
77	Return,
78	Begin,
79	Lambda,
80	Quote,
81	SyntaxRules,
82	DefineSyntax,
83	Ellipses,
84	Set,
85	Require,
86	CharacterLiteral(char),
87	DatumComment,
88	Comment,
89	BooleanLiteral(bool),
90	Identifier(S),
91	Keyword(S),
92	Number(Box<NumberLiteral>),
93	// TODO: Consider using Arc<String> here instead to save
94	// on copies
95	StringLiteral(Arc<String>),
96	Dot,
97	Error,
98	}
99
100	impl<T> TokenType<T> {
101	pub fn identifier_mut(&mut self) -> Option<&mut T> {	341,030✔
102	if let Self::Identifier(i) = self {	682,060✔
103	Some(i)	×
104	} else {
105	None	×
106	}
107	}
108	}
109
110	#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
111	pub enum NumberLiteral {
112	Real(RealLiteral),
113	Complex(RealLiteral, RealLiteral),
114	}
115
116	impl Display for NumberLiteral {
117	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	68✔
118	match self {	68✔
119	NumberLiteral::Real(r) => r.fmt(f),	68✔
120	NumberLiteral::Complex(re, im) => {	×
121	if im.is_negative() {	×
122	write!(f, "{re}{im}i")	×
123	} else {
124	write!(f, "{re}+{im}i")	×
125	}
126	}
127	}
128	}
129	}
130
131	impl<S> From<NumberLiteral> for TokenType<S> {
132	fn from(n: NumberLiteral) -> Self {	44,339✔
133	TokenType::Number(Box::new(n))	44,339✔
134	}
135	}
136
137	#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
138	pub enum RealLiteral {
139	Int(IntLiteral),
140	Rational(IntLiteral, IntLiteral),
141	Float(f64),
142	}
143
144	impl RealLiteral {
145	fn is_negative(&self) -> bool {	×
146	match self {	×
147	RealLiteral::Int(i) => i.is_negative(),	×
148	RealLiteral::Rational(n, _) => n.is_negative(),	×
149	RealLiteral::Float(f) => f.is_sign_negative(),	×
150	}
151	}
152	}
153
154	impl From<RealLiteral> for NumberLiteral {
155	fn from(value: RealLiteral) -> Self {	32,543✔
156	NumberLiteral::Real(value).into()	32,543✔
157	}
158	}
159
160	impl<S> From<RealLiteral> for TokenType<S> {
161	fn from(value: RealLiteral) -> Self {	11,927✔
162	NumberLiteral::Real(value).into()	11,927✔
163	}
164	}
165
166	impl From<f64> for RealLiteral {
167	fn from(value: f64) -> RealLiteral {	×
168	RealLiteral::Float(value)	×
169	}
170	}
171
172	impl From<isize> for RealLiteral {
173	fn from(value: isize) -> RealLiteral {	×
174	RealLiteral::Int(IntLiteral::Small(value))	×
175	}
176	}
177
178	impl From<Rational32> for RealLiteral {
179	fn from(value: Rational32) -> RealLiteral {	×
180	RealLiteral::Rational(
181	(*value.numer() as isize).into(),	×
182	(*value.denom() as isize).into(),	×
183	)
184	}
185	}
186
187	impl Display for RealLiteral {
188	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	94✔
189	match self {	94✔
190	RealLiteral::Int(i) => i.fmt(f),	68✔
191	RealLiteral::Rational(n, d) => write!(f, "{n}/{d}"),	×
192	RealLiteral::Float(x) => {	26✔
193	if x.is_nan() {	26✔
194	write!(f, "{}", lexer::NAN)	×
195	} else if x.is_infinite() && x.is_sign_negative() {	26✔
196	write!(f, "{}", lexer::NEG_INFINITY)	×
197	} else if x.is_infinite() {	26✔
198	write!(f, "{}", lexer::INFINITY)	×
199	} else {
200	x.fmt(f)	26✔
201	}
202	}
203	}
204	}
205	}
206
207	#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
208	pub enum IntLiteral {
209	Small(isize),
210	Big(Box<BigInt>),
211	}
212
213	impl IntLiteral {
214	fn is_negative(&self) -> bool {	×
215	match self {	×
216	IntLiteral::Small(i) => i.is_negative(),	×
217	IntLiteral::Big(i) => i.is_negative(),	×
218	}
219	}
220	}
221
222	impl FromStr for IntLiteral {
223	type Err = <num::BigInt as FromStr>::Err;
224
225	fn from_str(s: &str) -> Result<Self, Self::Err> {	32,495✔
226	s.parse::<isize>().map(IntLiteral::Small).or_else(\|_\| {	32,538✔
227	s.parse::<num::BigInt>()	43✔
228	.map(\|b\| IntLiteral::Big(Box::new(b)))	127✔
229	})
230	}
231	}
232
233	impl Display for IntLiteral {
234	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	68✔
235	match self {	68✔
236	Self::Small(s) => write!(f, "{s}"),	68✔
237	Self::Big(b) => write!(f, "{b}"),	×
238	}
239	}
240	}
241
242	impl<S> From<IntLiteral> for TokenType<S> {
243	fn from(value: IntLiteral) -> Self {	8,179✔
244	RealLiteral::Int(value).into()	8,179✔
245	}
246	}
247
248	impl From<IntLiteral> for RealLiteral {
249	fn from(value: IntLiteral) -> Self {	10✔
250	RealLiteral::Int(value)	10✔
251	}
252	}
253
254	impl From<IntLiteral> for BigInt {
255	fn from(v: IntLiteral) -> BigInt {	×
256	match v {	×
257	IntLiteral::Small(x) => x.into(),	×
258	IntLiteral::Big(x) => *x,	×
259	}
260	}
261	}
262
263	impl From<isize> for IntLiteral {
264	fn from(value: isize) -> Self {	×
265	IntLiteral::Small(value)	×
266	}
267	}
268
269	impl From<BigInt> for IntLiteral {
270	fn from(value: BigInt) -> Self {	×
271	IntLiteral::Big(Box::new(value))	×
272	}
273	}
274
275	impl<'a> TokenType<Cow<'a, str>> {
276	pub fn open_span(mut span: Span, paren_mod: Option<ParenMod>) -> Span {	×
277	let offset = match paren_mod {	×
278	Some(ParenMod::Vector) => 1,	×
279	Some(ParenMod::Bytes) => 3,	×
280	None => 0,	×
281	};
282
283	span.start += offset;	×
284
285	span	×
286	}
287
NEW 288	pub fn to_owned<T: From<Cow<'a, str>>>(self) -> TokenType<T> {	×
289	match self {	×
290	TokenType::Identifier(i) => TokenType::Identifier(i.into()),	×
291	TokenType::Keyword(i) => TokenType::Keyword(i.into()),	×
292	OpenParen(p, m) => OpenParen(p, m),	×
293	CloseParen(p) => CloseParen(p),	×
294	CharacterLiteral(x) => CharacterLiteral(x),	×
295	BooleanLiteral(x) => BooleanLiteral(x),	×
296	Number(x) => Number(x),	×
297	StringLiteral(x) => StringLiteral(x),	×
298	QuoteTick => QuoteTick,	×
299	Unquote => Unquote,	×
300	QuasiQuote => QuasiQuote,	×
301	UnquoteSplice => UnquoteSplice,	×
302	Error => Error,	×
303	Comment => Comment,	×
304	DatumComment => DatumComment,	×
305	If => If,	×
306	Define => Define,	×
307	Let => Let,	×
308	TestLet => TestLet,	×
309	Return => Return,	×
310	Begin => Begin,	×
311	Lambda => Lambda,	×
312	Quote => Quote,	×
313	DefineSyntax => DefineSyntax,	×
314	SyntaxRules => SyntaxRules,	×
315	Ellipses => Ellipses,	×
316	Set => Set,	×
317	Require => Require,	×
318	QuasiQuoteSyntax => QuasiQuoteSyntax,	×
319	UnquoteSyntax => UnquoteSyntax,	×
320	QuoteSyntax => QuoteSyntax,	×
321	UnquoteSpliceSyntax => UnquoteSpliceSyntax,	×
322	Dot => Dot,	×
323	}
324	}
325
326	pub fn map<T>(self, mut func: impl FnMut(Cow<'a, str>) -> T) -> TokenType<T> {	2,230,575✔
327	match self {	2,230,575✔
328	TokenType::Identifier(i) => TokenType::Identifier(func(i)),	766,959✔
329	TokenType::Keyword(i) => TokenType::Keyword(func(i)),	7,528✔
330	OpenParen(p, m) => OpenParen(p, m),	560,813✔
331	CloseParen(p) => CloseParen(p),	560,793✔
332	CharacterLiteral(x) => CharacterLiteral(x),	948✔
333	BooleanLiteral(x) => BooleanLiteral(x),	8,864✔
334	Number(x) => Number(x),	32,642✔
335	StringLiteral(x) => StringLiteral(x),	24,819✔
336	QuoteTick => QuoteTick,	23,757✔
337	Unquote => Unquote,	18,994✔
338	QuasiQuote => QuasiQuote,	7,468✔
339	UnquoteSplice => UnquoteSplice,	2,161✔
340	Error => Error,	1✔
341	Comment => Comment,	55,326✔
342	DatumComment => DatumComment,	7✔
343	If => If,	15,777✔
344	Define => Define,	52,415✔
345	Let => Let,	12,510✔
346	TestLet => TestLet,	1,676✔
347	Return => Return,	217✔
348	Begin => Begin,	8,858✔
349	Lambda => Lambda,	19,573✔
350	Quote => Quote,	12,214✔
351	DefineSyntax => DefineSyntax,	4,007✔
352	SyntaxRules => SyntaxRules,	3,677✔
353	Ellipses => Ellipses,	18,041✔
354	Set => Set,	7,110✔
355	Require => Require,	2,321✔
356	QuasiQuoteSyntax => QuasiQuoteSyntax,	3✔
357	UnquoteSyntax => UnquoteSyntax,	2✔
358	QuoteSyntax => QuoteSyntax,	11✔
359	UnquoteSpliceSyntax => UnquoteSpliceSyntax,	1✔
360	Dot => Dot,	1,082✔
361	}
362	}
363	}
364
365	fn character_special_display(c: char, f: &mut fmt::Formatter) -> fmt::Result {	×
366	match c {	×
367	' ' => write!(f, "#\\SPACE"),	×
368	'\t' => write!(f, "#\\TAB"),	×
369	'\n' => write!(f, "#\\NEWLINE"),	×
370	'\r' => write!(f, "#\\RETURN"),	×
371	c if c.is_control() \|\| c.is_whitespace() => {	×
372	write!(f, "#\\{}", c.escape_unicode())
373	}
374	// '\"' => write!(f, "#\\DOUBLE-QUOTE"),
375	// '\'' => write!(f, "#\\QUOTE"),
376	// '\\' => write!(f, "#\\BACKSLASH"),
377	_ => write!(f, "#\\{c}"),	×
378	}
379	}
380
381	impl<T: Display> fmt::Display for TokenType<T> {
382	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {	38,614✔
383	match self {	38,614✔
384	OpenParen(p, m) => {	×
385	if let Some(m) = m {	×
386	m.fmt(f)?;	×
387	}
388
389	write!(f, "{}", p.open())	×
390	}
391	CloseParen(p) => write!(f, "{}", p.close()),	×
392	CharacterLiteral(x) => character_special_display(*x, f),	×
393	BooleanLiteral(x) => write!(f, "#{x}"),	3✔
394	Identifier(x) => write!(f, "{x}"),	38,543✔
395	Number(x) => write!(f, "{x}"),	68✔
396	StringLiteral(x) => write!(f, "\"{x}\""),	×
397	Keyword(x) => write!(f, "{x}"),	×
398	QuoteTick => write!(f, "'"),	×
399	Unquote => write!(f, ","),	×
400	QuasiQuote => write!(f, "`"),	×
401	UnquoteSplice => write!(f, ",@"),	×
402	QuoteSyntax => write!(f, "#'"),	×
403	QuasiQuoteSyntax => write!(f, "#`"),	×
404	UnquoteSyntax => write!(f, "#,"),	×
405	UnquoteSpliceSyntax => write!(f, "#,@"),	×
406	Error => write!(f, "error"),	×
407	DatumComment => write!(f, "#;"),	×
408	Comment => write!(f, ""),	×
409	If => write!(f, "if"),	×
410	Define => write!(f, "define"),	×
411	Let => write!(f, "let"),	×
412	TestLet => write!(f, "%plain-let"),	×
413	Return => write!(f, "return!"),	×
414	Begin => write!(f, "begin"),	×
415	Lambda => write!(f, "lambda"),	×
416	Quote => write!(f, "quote"),	×
417	DefineSyntax => write!(f, "define-syntax"),	×
418	SyntaxRules => write!(f, "syntax-rules"),	×
419	Ellipses => write!(f, "..."),	×
420	Set => write!(f, "set!"),	×
421	Require => write!(f, "require"),	×
422	Dot => write!(f, "."),	×
423	}
424	}
425	}
426
427	#[derive(Debug, Clone, PartialEq)]
428	pub struct Token<'a, T> {
429	pub ty: TokenType<T>,
430	pub source: &'a str,
431	pub span: Span,
432	}
433
434	impl<'a, T> Token<'a, T> {
435	pub const fn new(	2,230,769✔
436	ty: TokenType<T>,
437	source: &'a str,
438	range: ops::Range<usize>,
439	source_id: Option<SourceId>,
440	) -> Self {
441	Self {
442	ty,
443	source,
444	span: Span::new(range.start, range.end, source_id),	2,230,769✔
445	}
446	}
447
448	pub fn typ(&self) -> &TokenType<T> {	×
449	&self.ty	×
450	}
451
452	pub const fn span(&self) -> Span {	×
453	self.span	×
454	}
455
456	pub const fn range(&self) -> ops::Range<usize> {	×
457	self.span.start()..self.span.end()	×
458	}
459
460	pub const fn source(&self) -> &'a str {	29,746✔
461	self.source	29,746✔
462	}
463	}
464
465	impl<T> From<Token<'_, T>> for Span {
466	fn from(token: Token<'_, T>) -> Self {	×
467	token.span()	×
468	}
469	}
470
471	impl<T> From<&Token<'_, T>> for Span {
472	fn from(token: &Token<'_, T>) -> Self {	×
473	token.span()	×
474	}
475	}
476
477	impl<T> From<Token<'_, T>> for ops::Range<usize> {
478	fn from(token: Token<'_, T>) -> Self {	×
479	token.span().into()	×
480	}
481	}
482
483	impl<T> From<&Token<'_, T>> for ops::Range<usize> {
484	fn from(token: &Token<'_, T>) -> Self {	×
485	token.span().into()	×
486	}
487	}
488
489	impl<T> From<Token<'_, T>> for (usize, usize) {
490	fn from(token: Token<'_, T>) -> Self {	×
491	token.span().into()	×
492	}
493	}
494
495	impl<T> From<&Token<'_, T>> for (usize, usize) {
496	fn from(token: &Token<'_, T>) -> Self {	×
497	token.span().into()	×
498	}
499	}
500
501	impl<T> From<Token<'_, T>> for [usize; 2] {
502	fn from(token: Token<'_, T>) -> Self {	×
503	token.span().into()	×
504	}
505	}
506
507	impl<T> From<&Token<'_, T>> for [usize; 2] {
508	fn from(token: &Token<'_, T>) -> Self {	×
509	token.span().into()	×
510	}
511	}
512
513	impl<T> Display for Token<'_, T> {
514	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {	×
515	write!(f, "{} @ {:?}", self.source, self.span)	×
516	}
517	}

mattwparas / steel / 13615020054

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous