23096627001

Committed 14 Mar 2026 09:22PM UTC coverage: 34.925% (+0.09%) from 34.84%

Build # 23096627001

Build Type

Pull #88

github

Committed by

web-flow

Commit Message

Merge 41bdd8608 into aad65462a

Pull Request Pull Request #88: Various package improvements

Coverage Stats

266 of 1054 branches covered (25.24%)

Branch coverage included in aggregate %.

879 of 2533 new or added lines in 17 files covered. (34.7%)

16 existing lines in 6 files now uncovered.

1109 of 2883 relevant lines covered (38.47%)

84.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.61

/src/parser/parser.ts

/*
* Full disclosure: some of the functions and general layout of the file is
* from my own implementation of a parser
* in Rust.
* https://github.com/Fidget-Spinner/crafting_interpreters/blob/main/rust/src/parser.rs
*
* That is in turn an implementation of the book "Crafting Interpreters" by
* Robert Nystrom, which implements an interpreter in Java.
* https://craftinginterpreters.com/parsing-expressions.html.
* I've included the MIT license that code snippets from
* the book is licensed under down below. See
* https://github.com/munificent/craftinginterpreters/blob/master/LICENSE
*
*
* My changes:
*   - The book was written in Java. I have written this in TypeScript.
*   - My Rust implementation uses pattern matching, but the visitor pattern is
*     used here.
*   - Additionally, the production rules are completely different
*     from the book as a whole different language is being parsed.
*
*
    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
    deal in the Software without restriction, including without limitation the
    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    sell copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.
**/

import { ExprNS, FunctionParam, StmtNS } from "../ast-types";
import { SPECIAL_IDENTIFIER_TOKENS, Token } from "../tokenizer/tokenizer";
import { TokenType } from "../tokens";
import { ParserErrors } from "./errors";

type Expr = ExprNS.Expr;
type Stmt = StmtNS.Stmt;

const PSEUD_NAMES = [TokenType.TRUE, TokenType.FALSE, TokenType.NONE];

export class Parser {
  private readonly source: string;
  private readonly tokens: Token[];
  private current: number;

  constructor(source: string, tokens: Token[]) {
    this.source = source;
    this.tokens = tokens;
    this.current = 0;
  }

  // Consumes tokens while tokenTypes matches.
  private match(...tokenTypes: TokenType[]): boolean {
    for (const tokenType of tokenTypes) {
      if (this.check(tokenType)) {
        this.advance();
        return true;
      }
    }
    return false;
  }

  private check(...type: TokenType[]): boolean {
    if (this.isAtEnd()) {
      return false;
    }
    for (const tokenType of type) {
      if (this.peek().type === tokenType) {
        return true;
      }
    }
    return false;
  }

  private advance(): Token {
    if (!this.isAtEnd()) {
      this.current += 1;
    }
    return this.previous();
  }

  private isAtEnd(): boolean {
    return this.peek().type === TokenType.ENDMARKER;
  }

  private peek(): Token {
    return this.tokens[this.current];
  }

  private previous(): Token {
    return this.tokens[this.current - 1];
  }

  private consume(type: TokenType, message: string): Token {
    if (this.check(type)) return this.advance();
    const token = this.tokens[this.current];
    throw new ParserErrors.ExpectedTokenError(this.source, token, message);
  }

  private synchronize() {
    this.advance();
    while (!this.isAtEnd()) {
      if (this.match(TokenType.NEWLINE)) {
        return false;
      }
      if (
        this.match(
          TokenType.FOR,
          TokenType.WHILE,
          TokenType.DEF,
          TokenType.IF,
          TokenType.ELIF,
          TokenType.ELSE,
          TokenType.RETURN,
        )
      ) {
        return true;
      }
      this.advance();
    }
    return false;
  }

  parse(): Stmt {
    return this.file_input();
    // return this.expression();
  }

  //// THE NAMES OF THE FOLLOWING FUNCTIONS FOLLOW THE PRODUCTION RULES IN THE GRAMMAR.
  //// HENCE THEIR NAMES MIGHT NOT BE COMPLIANT WITH CAMELCASE
  private file_input(): Stmt {
    const startToken = this.peek();
    const statements: Stmt[] = [];
    while (!this.isAtEnd()) {
      if (this.match(TokenType.NEWLINE) || this.match(TokenType.DEDENT)) {
        continue;
      }
      statements.push(this.stmt());
    }
    const endToken = this.peek();
    return new StmtNS.FileInput(startToken, endToken, statements, []);
  }

  private stmt(): Stmt {
    if (this.check(TokenType.DEF, TokenType.FOR, TokenType.IF, TokenType.WHILE)) {
      return this.compound_stmt();
    } else if (
      this.check(
        TokenType.NAME,
        ...PSEUD_NAMES,
        TokenType.NUMBER,
        TokenType.PASS,
        TokenType.BREAK,
        TokenType.CONTINUE,
        TokenType.MINUS,
        TokenType.PLUS,
        TokenType.INDENT,
        TokenType.DEDENT,
        TokenType.RETURN,
        TokenType.FROM,
        TokenType.GLOBAL,
        TokenType.NONLOCAL,
        TokenType.ASSERT,
        TokenType.LPAR,
        TokenType.LSQB,
        TokenType.STRING,
        TokenType.BIGINT,
        ...SPECIAL_IDENTIFIER_TOKENS,
      )
    ) {
      return this.simple_stmt();
    }
    const startToken = this.peek();
    const endToken = this.synchronize() ? this.previous() : this.peek();
    try {
      this.parse_invalid(startToken, endToken);
    } catch (e) {
      if (e instanceof ParserErrors.BaseParserError) {
        throw e;
      }
    }
    throw new ParserErrors.GenericUnexpectedSyntaxError(
      startToken.line,
      startToken.col,
      this.source,
      startToken.indexInSource,
    );
  }

  private compound_stmt(): Stmt {
    if (this.match(TokenType.IF)) {
      return this.if_stmt();
    } else if (this.match(TokenType.WHILE)) {
      return this.while_stmt();
    } else if (this.match(TokenType.FOR)) {
      return this.for_stmt();
    } else if (this.match(TokenType.DEF)) {
      return this.funcdef();
    }
    throw new Error("Unreachable code path");
  }

  private if_stmt(): Stmt {
    const startToken = this.previous();
    const start = this.previous();
    const cond = this.test();
    this.consume(TokenType.COLON, "Expected ':' after if");
    const block = this.suite();
    let elseStmt = null;
    if (this.match(TokenType.ELIF)) {
      elseStmt = [this.if_stmt()];
    } else if (this.match(TokenType.ELSE)) {
      this.consume(TokenType.COLON, "Expect ':' after else");
      elseStmt = this.suite();
    } else {
      throw new ParserErrors.NoElseBlockError(this.source, start);
    }
    const endToken = this.previous();
    return new StmtNS.If(startToken, endToken, cond, block, elseStmt);
  }

  private while_stmt(): Stmt {
    const startToken = this.peek();
    const cond = this.test();
    this.consume(TokenType.COLON, "Expected ':' after while");
    const block = this.suite();
    const endToken = this.previous();
    return new StmtNS.While(startToken, endToken, cond, block);
  }

  private for_stmt(): Stmt {
    const startToken = this.peek();
    const target = this.advance();
    this.consume(TokenType.IN, "Expected in after for");
    const iter = this.test();
    this.consume(TokenType.COLON, "Expected ':' after for");
    const block = this.suite();
    const endToken = this.previous();
    return new StmtNS.For(startToken, endToken, target, iter, block);
  }

  private funcdef(): Stmt {
    const startToken = this.peek();
    const name = this.advance();
    const args = this.parameters();
    this.consume(TokenType.COLON, "Expected ':' after def");
    const block = this.suite();
    const endToken = this.previous();
    return new StmtNS.FunctionDef(startToken, endToken, name, args, block, []);
  }

  private simple_stmt(): Stmt {
    const startToken = this.peek();
    let res = null;
    if (this.match(TokenType.INDENT)) {
      res = new StmtNS.Indent(startToken, startToken);
    } else if (this.match(TokenType.DEDENT)) {
      res = new StmtNS.Dedent(startToken, startToken);
    } else if (this.match(TokenType.PASS)) {
      res = new StmtNS.Pass(startToken, startToken);
    } else if (this.match(TokenType.BREAK)) {
      res = new StmtNS.Break(startToken, startToken);
    } else if (this.match(TokenType.CONTINUE)) {
      res = new StmtNS.Continue(startToken, startToken);
    } else if (this.match(TokenType.RETURN)) {
      res = new StmtNS.Return(
        startToken,
        startToken,
        this.check(TokenType.NEWLINE) ? null : this.test(),
      );
    } else if (this.match(TokenType.FROM)) {
      res = this.import_from();
    } else if (this.match(TokenType.GLOBAL)) {
      res = new StmtNS.Global(startToken, startToken, this.advance());
    } else if (this.match(TokenType.NONLOCAL)) {
      res = new StmtNS.NonLocal(startToken, startToken, this.advance());
    } else if (this.match(TokenType.ASSERT)) {
      res = new StmtNS.Assert(startToken, startToken, this.test());
    } else if (
      this.check(
        TokenType.NAME,
        TokenType.LPAR,
        TokenType.LSQB,
        TokenType.NUMBER,
        TokenType.STRING,
        TokenType.BIGINT,
        TokenType.MINUS,
        TokenType.PLUS,
        ...SPECIAL_IDENTIFIER_TOKENS,
      )
    ) {
      const expr = this.test();

      if (this.check(TokenType.COLON)) {
        if (!(expr instanceof ExprNS.Variable)) {
          throw new ParserErrors.InvalidAssignmentError(this.source, startToken);
        }
        this.advance();
        const ann = this.test();
        this.consume(TokenType.EQUAL, "Expect equal in annotated assignment");
        const value = this.test();
        res = new StmtNS.AnnAssign(startToken, this.previous(), expr, value, ann);
      } else if (this.check(TokenType.EQUAL)) {
        if (!(expr instanceof ExprNS.Variable || expr instanceof ExprNS.Subscript)) {
          throw new ParserErrors.InvalidAssignmentError(this.source, startToken);
        }
        this.advance();
        const value = this.test();
        res = new StmtNS.Assign(startToken, this.previous(), expr, value);
      } else {
        res = new StmtNS.SimpleExpr(startToken, this.previous(), expr);
      }
      // res = new StmtNS.SimpleExpr(startToken, startToken, expr);
    } else {
      throw new Error("Unreachable code path");
    }
    this.consume(TokenType.NEWLINE, "Expected newline");
    return res;
  }

  private import_from(): Stmt {
    const startToken = this.previous();
    const module = this.advance();
    this.consume(TokenType.IMPORT, "Expected import keyword");

    const names: Token[] = [];
    let useParens = false;

    if (this.check(TokenType.LPAR)) {
      this.consume(TokenType.LPAR, "Expected '(' after import");
      useParens = true;
    }

    names.push(this.consume(TokenType.NAME, "Expected name to import"));
    while (this.match(TokenType.COMMA)) {
      names.push(this.consume(TokenType.NAME, "Expected name after comma"));
    }

    if (useParens) {
      this.consume(TokenType.RPAR, "Expected ')' after import");
    }

    return new StmtNS.FromImport(startToken, this.previous(), module, names);
  }

  private parameters(): FunctionParam[] {
    this.consume(TokenType.LPAR, "Expected opening parentheses");
    const res = this.varparamslist();
    this.consume(TokenType.RPAR, "Expected closing parentheses");
    return res;
  }

  private test(): Expr {
    if (this.match(TokenType.LAMBDA)) {
      return this.lambdef();
    } else {
      const startToken = this.peek();
      const consequent = this.or_test();
      if (this.match(TokenType.IF)) {
        const predicate = this.or_test();
        this.consume(TokenType.ELSE, "Expected else");
        const alternative = this.test();
        return new ExprNS.Ternary(startToken, this.previous(), predicate, consequent, alternative);
      }
      return consequent;
    }
  }

  private lambdef(): Expr {
    const startToken = this.previous();
    const args = this.varparamslist();
    if (this.match(TokenType.COLON)) {
      const test = this.test();
      return new ExprNS.Lambda(startToken, this.previous(), args, test);
    } else if (this.match(TokenType.DOUBLECOLON)) {
      const block = this.suite();
      return new ExprNS.MultiLambda(startToken, this.previous(), args, block, []);
    }
    this.consume(TokenType.COLON, "Expected ':' after lambda");
    throw new Error("unreachable code path");
  }

  private suite(): Stmt[] {
    const stmts = [];
    if (this.match(TokenType.NEWLINE)) {
      this.consume(TokenType.INDENT, "Expected indent");
      while (!this.match(TokenType.DEDENT)) {
        stmts.push(this.stmt());
      }
    }
    return stmts;
  }

  private varparamslist(): FunctionParam[] {
    const params = [];
    while (!this.check(TokenType.COLON) && !this.check(TokenType.RPAR)) {
      if (this.match(TokenType.STAR)) {
        const name = this.consume(
          TokenType.NAME,
          "Expected a proper identifier after * in parameter",
        );
        params.push({ ...name, isStarred: true });
      } else {
        const name = this.consume(TokenType.NAME, "Expected a proper identifier in parameter");
        params.push({ ...name, isStarred: false });
      }
      if (!this.match(TokenType.COMMA)) {
        break;
      }
    }
    return params;
  }

  private or_test(): Expr {
    const startToken = this.peek();
    let expr = this.and_test();
    while (this.match(TokenType.OR)) {
      const operator = this.previous();
      const right = this.and_test();
      expr = new ExprNS.BoolOp(startToken, this.previous(), expr, operator, right);
    }
    return expr;
  }

  private and_test(): Expr {
    const startToken = this.peek();
    let expr = this.not_test();
    while (this.match(TokenType.AND)) {
      const operator = this.previous();
      const right = this.not_test();
      expr = new ExprNS.BoolOp(startToken, this.previous(), expr, operator, right);
    }
    return expr;
  }

  private not_test(): Expr {
    const startToken = this.peek();
    if (this.match(TokenType.NOT, TokenType.BANG)) {
      const operator = this.previous();
      return new ExprNS.Unary(startToken, this.previous(), operator, this.not_test());
    }
    return this.comparison();
  }

  private comparison(): Expr {
    const startToken = this.peek();
    let expr = this.arith_expr();
    // @TODO: Add the rest of the comparisons
    while (
      this.match(
        TokenType.LESS,
        TokenType.GREATER,
        TokenType.DOUBLEEQUAL,
        TokenType.GREATEREQUAL,
        TokenType.LESSEQUAL,
        TokenType.NOTEQUAL,
        TokenType.IS,
        TokenType.ISNOT,
        TokenType.IN,
        TokenType.NOTIN,
      )
    ) {
      const operator = this.previous();
      const right = this.arith_expr();
      expr = new ExprNS.Compare(startToken, this.previous(), expr, operator, right);
    }
    return expr;
  }

  private arith_expr(): Expr {
    const startToken = this.peek();
    let expr = this.term();
    while (this.match(TokenType.PLUS, TokenType.MINUS)) {
      const token = this.previous();
      const right = this.term();
      expr = new ExprNS.Binary(startToken, this.previous(), expr, token, right);
    }
    return expr;
  }

  private term(): Expr {
    const startToken = this.peek();
    let expr = this.factor();
    while (this.match(TokenType.STAR, TokenType.SLASH, TokenType.PERCENT, TokenType.DOUBLESLASH)) {
      const token = this.previous();
      const right = this.factor();
      expr = new ExprNS.Binary(startToken, this.previous(), expr, token, right);
    }
    return expr;
  }

  private factor(): Expr {
    const startToken = this.peek();
    if (this.match(TokenType.PLUS, TokenType.MINUS)) {
      const op = this.previous();
      const factor = this.factor();
      const endToken = this.previous();
      return new ExprNS.Unary(startToken, endToken, op, factor);
    }
    return this.power();
  }

  private power(): Expr {
    const startToken = this.peek();
    const expr = this.atom_expr();
    if (this.match(TokenType.DOUBLESTAR)) {
      const token = this.previous();
      const right = this.factor();
      const endToken = this.previous();
      return new ExprNS.Binary(startToken, endToken, expr, token, right);
    }
    return expr;
  }

  private atom_expr(): Expr {
    let startToken = this.peek();
    let ato = this.atom();
    while (this.check(TokenType.LPAR, TokenType.LSQB)) {
      if (this.match(TokenType.LPAR)) {
        const args = this.arglist();
        const endToken = this.previous();
        ato = new ExprNS.Call(startToken, endToken, ato, args);
      } else if (this.match(TokenType.LSQB)) {
        const index = this.test();
        const endToken = this.previous();
        this.consume(TokenType.RSQB, "Expected closing ']'");
        ato = new ExprNS.Subscript(startToken, endToken, ato, index);
      }
      startToken = this.peek();
    }
    return ato;
  }

  private arglist(): Expr[] {
    const args = [];
    while (!this.check(TokenType.RPAR)) {
      const startToken = this.peek();
      if (this.match(TokenType.STAR)) {
        const arg = this.test();
        args.push(new ExprNS.Starred(startToken, this.previous(), arg));
      } else {
        args.push(this.test());
      }
      if (!this.match(TokenType.COMMA)) {
        break;
      }
    }
    this.consume(TokenType.RPAR, "Expected closing ')' after function application");
    return args;
  }

  private list_expr(): Expr[] {
    const elements: Expr[] = [];
    while (!this.check(TokenType.RSQB)) {
      const element = this.test();
      elements.push(element);
      if (!this.match(TokenType.COMMA)) {
        break;
      }
    }
    this.consume(TokenType.RSQB, "Expected closing ']'");
    return elements;
  }

  private atom(): Expr {
    const startToken = this.peek();
    if (this.match(TokenType.TRUE)) return new ExprNS.Literal(startToken, this.previous(), true);
    if (this.match(TokenType.FALSE)) return new ExprNS.Literal(startToken, this.previous(), false);
    if (this.match(TokenType.NONE)) return new ExprNS.None(startToken, this.previous());
    if (this.match(TokenType.STRING)) {
      return new ExprNS.Literal(startToken, this.previous(), this.previous().lexeme);
    }
    if (this.match(TokenType.NUMBER)) {
      return new ExprNS.Literal(
        startToken,
        this.previous(),
        Number(this.previous().lexeme.replace(/_/g, "")),
      );
    }
    if (this.match(TokenType.BIGINT)) {
      return new ExprNS.BigIntLiteral(startToken, this.previous(), this.previous().lexeme);
    }
    if (this.match(TokenType.COMPLEX)) {
      return new ExprNS.Complex(startToken, this.previous(), this.previous().lexeme);
    }

    if (this.match(TokenType.NAME, ...PSEUD_NAMES)) {
      return new ExprNS.Variable(startToken, this.previous(), this.previous());
    }

    if (this.match(TokenType.LPAR)) {
      const expr = this.test();
      this.consume(TokenType.RPAR, "Expected closing ')'");
      return new ExprNS.Grouping(startToken, this.previous(), expr);
    }

    if (this.match(TokenType.LSQB)) {
      const elements = this.list_expr();
      return new ExprNS.List(startToken, this.previous(), elements);
    }
    const startTokenInvalid = this.peek();
    this.synchronize();
    throw new ParserErrors.GenericUnexpectedSyntaxError(
      startToken.line,
      startToken.col,
      this.source,
      startTokenInvalid.indexInSource,
    );
  }

  //// INVALID RULES
  private parse_invalid(_startToken: Token, _endToken: Token) {
    // @TODO invalid rules
  }
}

1	/*
2	* Full disclosure: some of the functions and general layout of the file is
3	* from my own implementation of a parser
4	* in Rust.
5	* https://github.com/Fidget-Spinner/crafting_interpreters/blob/main/rust/src/parser.rs
6	*
7	* That is in turn an implementation of the book "Crafting Interpreters" by
8	* Robert Nystrom, which implements an interpreter in Java.
9	* https://craftinginterpreters.com/parsing-expressions.html.
10	* I've included the MIT license that code snippets from
11	* the book is licensed under down below. See
12	* https://github.com/munificent/craftinginterpreters/blob/master/LICENSE
13	*
14	*
15	* My changes:
16	* - The book was written in Java. I have written this in TypeScript.
17	* - My Rust implementation uses pattern matching, but the visitor pattern is
18	* used here.
19	* - Additionally, the production rules are completely different
20	* from the book as a whole different language is being parsed.
21	*
22	*
23	Permission is hereby granted, free of charge, to any person obtaining a copy
24	of this software and associated documentation files (the "Software"), to
25	deal in the Software without restriction, including without limitation the
26	rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
27	sell copies of the Software, and to permit persons to whom the Software is
28	furnished to do so, subject to the following conditions:
29
30	The above copyright notice and this permission notice shall be included in
31	all copies or substantial portions of the Software.
32
33	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
38	FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
39	IN THE SOFTWARE.
40	**/
41
42	import { ExprNS, FunctionParam, StmtNS } from "../ast-types";	2✔
43	import { SPECIAL_IDENTIFIER_TOKENS, Token } from "../tokenizer/tokenizer";	2✔
44	import { TokenType } from "../tokens";	2✔
45	import { ParserErrors } from "./errors";	2✔
46
47	type Expr = ExprNS.Expr;
48	type Stmt = StmtNS.Stmt;
49
50	const PSEUD_NAMES = [TokenType.TRUE, TokenType.FALSE, TokenType.NONE];	2✔
51
52	export class Parser {	2✔
53	private readonly source: string;
54	private readonly tokens: Token[];
55	private current: number;
56
57	constructor(source: string, tokens: Token[]) {
58	this.source = source;	99✔
59	this.tokens = tokens;	99✔
60	this.current = 0;	99✔
61	}
62
63	// Consumes tokens while tokenTypes matches.
64	private match(...tokenTypes: TokenType[]): boolean {
65	for (const tokenType of tokenTypes) {	7,539✔
66	if (this.check(tokenType)) {	11,959✔
67	this.advance();	700✔
68	return true;	700✔
69	}
70	}
71	return false;	6,839✔
72	}
73
74	private check(...type: TokenType[]): boolean {
75	if (this.isAtEnd()) {	13,713!
NEW 76	return false;	×
77	}
78	for (const tokenType of type) {	13,713✔
79	if (this.peek().type === tokenType) {	16,592✔
80	return true;	1,576✔
81	}
82	}
83	return false;	12,137✔
84	}
85
86	private advance(): Token {
87	if (!this.isAtEnd()) {	1,176✔
88	this.current += 1;	1,176✔
89	}
90	return this.previous();	1,176✔
91	}
92
93	private isAtEnd(): boolean {
94	return this.peek().type === TokenType.ENDMARKER;	15,146✔
95	}
96
97	private peek(): Token {
98	return this.tokens[this.current];	35,551✔
99	}
100
101	private previous(): Token {
102	return this.tokens[this.current - 1];	2,272✔
103	}
104
105	private consume(type: TokenType, message: string): Token {
106	if (this.check(type)) return this.advance();	410✔
NEW 107	const token = this.tokens[this.current];	×
NEW 108	throw new ParserErrors.ExpectedTokenError(this.source, token, message);	×
109	}
110
111	private synchronize() {
NEW 112	this.advance();	×
NEW 113	while (!this.isAtEnd()) {	×
NEW 114	if (this.match(TokenType.NEWLINE)) {	×
UNCOV 115	return false;	×
116	}
NEW 117	if (	×
118	this.match(
119	TokenType.FOR,
120	TokenType.WHILE,
121	TokenType.DEF,
122	TokenType.IF,
123	TokenType.ELIF,
124	TokenType.ELSE,
125	TokenType.RETURN,
126	)
127	) {
NEW 128	return true;	×
129	}
NEW 130	this.advance();	×
131	}
NEW 132	return false;	×
133	}
134
135	parse(): Stmt {
136	return this.file_input();	99✔
137	// return this.expression();
138	}
139
140	//// THE NAMES OF THE FOLLOWING FUNCTIONS FOLLOW THE PRODUCTION RULES IN THE GRAMMAR.
141	//// HENCE THEIR NAMES MIGHT NOT BE COMPLIANT WITH CAMELCASE
142	private file_input(): Stmt {
143	const startToken = this.peek();	99✔
144	const statements: Stmt[] = [];	99✔
145	while (!this.isAtEnd()) {	99✔
146	if (this.match(TokenType.NEWLINE) \|\| this.match(TokenType.DEDENT)) {	158✔
147	continue;	24✔
148	}
149	statements.push(this.stmt());	134✔
150	}
151	const endToken = this.peek();	99✔
152	return new StmtNS.FileInput(startToken, endToken, statements, []);	99✔
153	}
154
155	private stmt(): Stmt {
156	if (this.check(TokenType.DEF, TokenType.FOR, TokenType.IF, TokenType.WHILE)) {	200✔
157	return this.compound_stmt();	30✔
158	} else if (	170✔
159	this.check(
160	TokenType.NAME,
161	...PSEUD_NAMES,
162	TokenType.NUMBER,
163	TokenType.PASS,
164	TokenType.BREAK,
165	TokenType.CONTINUE,
166	TokenType.MINUS,
167	TokenType.PLUS,
168	TokenType.INDENT,
169	TokenType.DEDENT,
170	TokenType.RETURN,
171	TokenType.FROM,
172	TokenType.GLOBAL,
173	TokenType.NONLOCAL,
174	TokenType.ASSERT,
175	TokenType.LPAR,
176	TokenType.LSQB,
177	TokenType.STRING,
178	TokenType.BIGINT,
179	...SPECIAL_IDENTIFIER_TOKENS,
180	)
181	) {
182	return this.simple_stmt();	170✔
183	}
NEW 184	const startToken = this.peek();	×
NEW 185	const endToken = this.synchronize() ? this.previous() : this.peek();	×
NEW 186	try {	×
NEW 187	this.parse_invalid(startToken, endToken);	×
188	} catch (e) {
NEW 189	if (e instanceof ParserErrors.BaseParserError) {	×
NEW 190	throw e;	×
191	}
192	}
NEW 193	throw new ParserErrors.GenericUnexpectedSyntaxError(	×
194	startToken.line,
195	startToken.col,
196	this.source,
197	startToken.indexInSource,
198	);
199	}
200
201	private compound_stmt(): Stmt {
202	if (this.match(TokenType.IF)) {	30✔
203	return this.if_stmt();	3✔
204	} else if (this.match(TokenType.WHILE)) {	27✔
205	return this.while_stmt();	1✔
206	} else if (this.match(TokenType.FOR)) {	26✔
207	return this.for_stmt();	1✔
208	} else if (this.match(TokenType.DEF)) {	25✔
209	return this.funcdef();	25✔
210	}
NEW 211	throw new Error("Unreachable code path");	×
212	}
213
214	private if_stmt(): Stmt {
215	const startToken = this.previous();	6✔
216	const start = this.previous();	6✔
217	const cond = this.test();	6✔
218	this.consume(TokenType.COLON, "Expected ':' after if");	6✔
219	const block = this.suite();	6✔
220	let elseStmt = null;	6✔
221	if (this.match(TokenType.ELIF)) {	6✔
222	elseStmt = [this.if_stmt()];	3✔
223	} else if (this.match(TokenType.ELSE)) {	3!
224	this.consume(TokenType.COLON, "Expect ':' after else");	3✔
225	elseStmt = this.suite();	3✔
226	} else {
NEW 227	throw new ParserErrors.NoElseBlockError(this.source, start);	×
228	}
229	const endToken = this.previous();	6✔
230	return new StmtNS.If(startToken, endToken, cond, block, elseStmt);	6✔
231	}
232
233	private while_stmt(): Stmt {
234	const startToken = this.peek();	1✔
235	const cond = this.test();	1✔
236	this.consume(TokenType.COLON, "Expected ':' after while");	1✔
237	const block = this.suite();	1✔
238	const endToken = this.previous();	1✔
239	return new StmtNS.While(startToken, endToken, cond, block);	1✔
240	}
241
242	private for_stmt(): Stmt {
243	const startToken = this.peek();	1✔
244	const target = this.advance();	1✔
245	this.consume(TokenType.IN, "Expected in after for");	1✔
246	const iter = this.test();	1✔
247	this.consume(TokenType.COLON, "Expected ':' after for");	1✔
248	const block = this.suite();	1✔
249	const endToken = this.previous();	1✔
250	return new StmtNS.For(startToken, endToken, target, iter, block);	1✔
251	}
252
253	private funcdef(): Stmt {
254	const startToken = this.peek();	25✔
255	const name = this.advance();	25✔
256	const args = this.parameters();	25✔
257	this.consume(TokenType.COLON, "Expected ':' after def");	25✔
258	const block = this.suite();	25✔
259	const endToken = this.previous();	25✔
260	return new StmtNS.FunctionDef(startToken, endToken, name, args, block, []);	25✔
261	}
262
263	private simple_stmt(): Stmt {
264	const startToken = this.peek();	170✔
265	let res = null;	170✔
266	if (this.match(TokenType.INDENT)) {	170!
NEW 267	res = new StmtNS.Indent(startToken, startToken);	×
268	} else if (this.match(TokenType.DEDENT)) {	170!
NEW 269	res = new StmtNS.Dedent(startToken, startToken);	×
270	} else if (this.match(TokenType.PASS)) {	170✔
271	res = new StmtNS.Pass(startToken, startToken);	9✔
272	} else if (this.match(TokenType.BREAK)) {	161!
NEW 273	res = new StmtNS.Break(startToken, startToken);	×
274	} else if (this.match(TokenType.CONTINUE)) {	161!
NEW 275	res = new StmtNS.Continue(startToken, startToken);	×
276	} else if (this.match(TokenType.RETURN)) {	161✔
277	res = new StmtNS.Return(	20✔
278	startToken,
279	startToken,
280	this.check(TokenType.NEWLINE) ? null : this.test(),	20!
281	);
282	} else if (this.match(TokenType.FROM)) {	141✔
283	res = this.import_from();	5✔
284	} else if (this.match(TokenType.GLOBAL)) {	136!
NEW 285	res = new StmtNS.Global(startToken, startToken, this.advance());	×
286	} else if (this.match(TokenType.NONLOCAL)) {	136✔
287	res = new StmtNS.NonLocal(startToken, startToken, this.advance());	5✔
288	} else if (this.match(TokenType.ASSERT)) {	131!
NEW 289	res = new StmtNS.Assert(startToken, startToken, this.test());	×
290	} else if (	131!
291	this.check(
292	TokenType.NAME,
293	TokenType.LPAR,
294	TokenType.LSQB,
295	TokenType.NUMBER,
296	TokenType.STRING,
297	TokenType.BIGINT,
298	TokenType.MINUS,
299	TokenType.PLUS,
300	...SPECIAL_IDENTIFIER_TOKENS,
301	)
302	) {
303	const expr = this.test();	131✔
304
305	if (this.check(TokenType.COLON)) {	131!
NEW 306	if (!(expr instanceof ExprNS.Variable)) {	×
NEW 307	throw new ParserErrors.InvalidAssignmentError(this.source, startToken);	×
308	}
NEW 309	this.advance();	×
NEW 310	const ann = this.test();	×
NEW 311	this.consume(TokenType.EQUAL, "Expect equal in annotated assignment");	×
NEW 312	const value = this.test();	×
NEW 313	res = new StmtNS.AnnAssign(startToken, this.previous(), expr, value, ann);	×
314	} else if (this.check(TokenType.EQUAL)) {	131✔
315	if (!(expr instanceof ExprNS.Variable \|\| expr instanceof ExprNS.Subscript)) {	30!
NEW 316	throw new ParserErrors.InvalidAssignmentError(this.source, startToken);	×
317	}
318	this.advance();	30✔
319	const value = this.test();	30✔
320	res = new StmtNS.Assign(startToken, this.previous(), expr, value);	30✔
321	} else {
322	res = new StmtNS.SimpleExpr(startToken, this.previous(), expr);	101✔
323	}
324	// res = new StmtNS.SimpleExpr(startToken, startToken, expr);
325	} else {
NEW 326	throw new Error("Unreachable code path");	×
327	}
328	this.consume(TokenType.NEWLINE, "Expected newline");	170✔
329	return res;	170✔
330	}
331
332	private import_from(): Stmt {
333	const startToken = this.previous();	5✔
334	const module = this.advance();	5✔
335	this.consume(TokenType.IMPORT, "Expected import keyword");	5✔
336
337	const names: Token[] = [];	5✔
338	let useParens = false;	5✔
339
340	if (this.check(TokenType.LPAR)) {	5✔
341	this.consume(TokenType.LPAR, "Expected '(' after import");	3✔
342	useParens = true;	3✔
343	}
344
345	names.push(this.consume(TokenType.NAME, "Expected name to import"));	5✔
346	while (this.match(TokenType.COMMA)) {	5✔
347	names.push(this.consume(TokenType.NAME, "Expected name after comma"));	4✔
348	}
349
350	if (useParens) {	5✔
351	this.consume(TokenType.RPAR, "Expected ')' after import");	3✔
352	}
353
354	return new StmtNS.FromImport(startToken, this.previous(), module, names);	5✔
355	}
356
357	private parameters(): FunctionParam[] {
358	this.consume(TokenType.LPAR, "Expected opening parentheses");	25✔
359	const res = this.varparamslist();	25✔
360	this.consume(TokenType.RPAR, "Expected closing parentheses");	25✔
361	return res;	25✔
362	}
363
364	private test(): Expr {
365	if (this.match(TokenType.LAMBDA)) {	262✔
366	return this.lambdef();	7✔
367	} else {
368	const startToken = this.peek();	255✔
369	const consequent = this.or_test();	255✔
370	if (this.match(TokenType.IF)) {	255✔
371	const predicate = this.or_test();	4✔
372	this.consume(TokenType.ELSE, "Expected else");	4✔
373	const alternative = this.test();	4✔
374	return new ExprNS.Ternary(startToken, this.previous(), predicate, consequent, alternative);	4✔
375	}
376	return consequent;	251✔
377	}
378	}
379
380	private lambdef(): Expr {
381	const startToken = this.previous();	7✔
382	const args = this.varparamslist();	7✔
383	if (this.match(TokenType.COLON)) {	7!
384	const test = this.test();	7✔
385	return new ExprNS.Lambda(startToken, this.previous(), args, test);	7✔
NEW 386	} else if (this.match(TokenType.DOUBLECOLON)) {	×
NEW 387	const block = this.suite();	×
NEW 388	return new ExprNS.MultiLambda(startToken, this.previous(), args, block, []);	×
389	}
NEW 390	this.consume(TokenType.COLON, "Expected ':' after lambda");	×
NEW 391	throw new Error("unreachable code path");	×
392	}
393
394	private suite(): Stmt[] {
395	const stmts = [];	36✔
396	if (this.match(TokenType.NEWLINE)) {	36✔
397	this.consume(TokenType.INDENT, "Expected indent");	36✔
398	while (!this.match(TokenType.DEDENT)) {	36✔
399	stmts.push(this.stmt());	66✔
400	}
401	}
402	return stmts;	36✔
403	}
404
405	private varparamslist(): FunctionParam[] {
406	const params = [];	32✔
407	while (!this.check(TokenType.COLON) && !this.check(TokenType.RPAR)) {	32✔
408	if (this.match(TokenType.STAR)) {	22!
NEW 409	const name = this.consume(	×
410	TokenType.NAME,
411	"Expected a proper identifier after * in parameter",
412	);
NEW 413	params.push({ ...name, isStarred: true });	×
414	} else {
415	const name = this.consume(TokenType.NAME, "Expected a proper identifier in parameter");	22✔
416	params.push({ ...name, isStarred: false });	22✔
417	}
418	if (!this.match(TokenType.COMMA)) {	22✔
419	break;	13✔
420	}
421	}
422	return params;	32✔
423	}
424
425	private or_test(): Expr {
426	const startToken = this.peek();	259✔
427	let expr = this.and_test();	259✔
428	while (this.match(TokenType.OR)) {	259✔
429	const operator = this.previous();	6✔
430	const right = this.and_test();	6✔
431	expr = new ExprNS.BoolOp(startToken, this.previous(), expr, operator, right);	6✔
432	}
433	return expr;	259✔
434	}
435
436	private and_test(): Expr {
437	const startToken = this.peek();	265✔
438	let expr = this.not_test();	265✔
439	while (this.match(TokenType.AND)) {	265✔
440	const operator = this.previous();	8✔
441	const right = this.not_test();	8✔
442	expr = new ExprNS.BoolOp(startToken, this.previous(), expr, operator, right);	8✔
443	}
444	return expr;	265✔
445	}
446
447	private not_test(): Expr {
448	const startToken = this.peek();	277✔
449	if (this.match(TokenType.NOT, TokenType.BANG)) {	277✔
450	const operator = this.previous();	4✔
451	return new ExprNS.Unary(startToken, this.previous(), operator, this.not_test());	4✔
452	}
453	return this.comparison();	273✔
454	}
455
456	private comparison(): Expr {
457	const startToken = this.peek();	273✔
458	let expr = this.arith_expr();	273✔
459	// @TODO: Add the rest of the comparisons
460	while (	273✔
461	this.match(
462	TokenType.LESS,
463	TokenType.GREATER,
464	TokenType.DOUBLEEQUAL,
465	TokenType.GREATEREQUAL,
466	TokenType.LESSEQUAL,
467	TokenType.NOTEQUAL,
468	TokenType.IS,
469	TokenType.ISNOT,
470	TokenType.IN,
471	TokenType.NOTIN,
472	)
473	) {
474	const operator = this.previous();	20✔
475	const right = this.arith_expr();	20✔
476	expr = new ExprNS.Compare(startToken, this.previous(), expr, operator, right);	20✔
477	}
478	return expr;	273✔
479	}
480
481	private arith_expr(): Expr {
482	const startToken = this.peek();	293✔
483	let expr = this.term();	293✔
484	while (this.match(TokenType.PLUS, TokenType.MINUS)) {	293✔
485	const token = this.previous();	33✔
486	const right = this.term();	33✔
487	expr = new ExprNS.Binary(startToken, this.previous(), expr, token, right);	33✔
488	}
489	return expr;	293✔
490	}
491
492	private term(): Expr {
493	const startToken = this.peek();	326✔
494	let expr = this.factor();	326✔
495	while (this.match(TokenType.STAR, TokenType.SLASH, TokenType.PERCENT, TokenType.DOUBLESLASH)) {	326✔
496	const token = this.previous();	9✔
497	const right = this.factor();	9✔
498	expr = new ExprNS.Binary(startToken, this.previous(), expr, token, right);	9✔
499	}
500	return expr;	326✔
501	}
502
503	private factor(): Expr {
504	const startToken = this.peek();	340✔
505	if (this.match(TokenType.PLUS, TokenType.MINUS)) {	340✔
506	const op = this.previous();	1✔
507	const factor = this.factor();	1✔
508	const endToken = this.previous();	1✔
509	return new ExprNS.Unary(startToken, endToken, op, factor);	1✔
510	}
511	return this.power();	339✔
512	}
513
514	private power(): Expr {
515	const startToken = this.peek();	339✔
516	const expr = this.atom_expr();	339✔
517	if (this.match(TokenType.DOUBLESTAR)) {	339✔
518	const token = this.previous();	4✔
519	const right = this.factor();	4✔
520	const endToken = this.previous();	4✔
521	return new ExprNS.Binary(startToken, endToken, expr, token, right);	4✔
522	}
523	return expr;	335✔
524	}
525
526	private atom_expr(): Expr {
527	let startToken = this.peek();	339✔
528	let ato = this.atom();	339✔
529	while (this.check(TokenType.LPAR, TokenType.LSQB)) {	339✔
530	if (this.match(TokenType.LPAR)) {	61!
531	const args = this.arglist();	61✔
532	const endToken = this.previous();	61✔
533	ato = new ExprNS.Call(startToken, endToken, ato, args);	61✔
NEW 534	} else if (this.match(TokenType.LSQB)) {	×
NEW 535	const index = this.test();	×
UNCOV 536	const endToken = this.previous();	×
UNCOV 537	this.consume(TokenType.RSQB, "Expected closing ']'");	×
NEW 538	ato = new ExprNS.Subscript(startToken, endToken, ato, index);	×
539	}
540	startToken = this.peek();	61✔
541	}
542	return ato;	339✔
543	}
544
545	private arglist(): Expr[] {
546	const args = [];	61✔
547	while (!this.check(TokenType.RPAR)) {	61✔
548	const startToken = this.peek();	52✔
549	if (this.match(TokenType.STAR)) {	52!
NEW 550	const arg = this.test();	×
NEW 551	args.push(new ExprNS.Starred(startToken, this.previous(), arg));	×
552	} else {
553	args.push(this.test());	52✔
554	}
555	if (!this.match(TokenType.COMMA)) {	52✔
556	break;	39✔
557	}
558	}
559	this.consume(TokenType.RPAR, "Expected closing ')' after function application");	61✔
560	return args;	61✔
561	}
562
563	private list_expr(): Expr[] {
NEW 564	const elements: Expr[] = [];	×
NEW 565	while (!this.check(TokenType.RSQB)) {	×
NEW 566	const element = this.test();	×
NEW 567	elements.push(element);	×
NEW 568	if (!this.match(TokenType.COMMA)) {	×
NEW 569	break;	×
570	}
571	}
NEW 572	this.consume(TokenType.RSQB, "Expected closing ']'");	×
NEW 573	return elements;	×
574	}
575
576	private atom(): Expr {
577	const startToken = this.peek();	339✔
578	if (this.match(TokenType.TRUE)) return new ExprNS.Literal(startToken, this.previous(), true);	339✔
579	if (this.match(TokenType.FALSE)) return new ExprNS.Literal(startToken, this.previous(), false);	337✔
580	if (this.match(TokenType.NONE)) return new ExprNS.None(startToken, this.previous());	336✔
581	if (this.match(TokenType.STRING)) {	332✔
582	return new ExprNS.Literal(startToken, this.previous(), this.previous().lexeme);	22✔
583	}
584	if (this.match(TokenType.NUMBER)) {	310✔
585	return new ExprNS.Literal(	10✔
586	startToken,
587	this.previous(),
588	Number(this.previous().lexeme.replace(/_/g, "")),
589	);
590	}
591	if (this.match(TokenType.BIGINT)) {	300✔
592	return new ExprNS.BigIntLiteral(startToken, this.previous(), this.previous().lexeme);	144✔
593	}
594	if (this.match(TokenType.COMPLEX)) {	156✔
595	return new ExprNS.Complex(startToken, this.previous(), this.previous().lexeme);	11✔
596	}
597
598	if (this.match(TokenType.NAME, ...PSEUD_NAMES)) {	145✔
599	return new ExprNS.Variable(startToken, this.previous(), this.previous());	135✔
600	}
601
602	if (this.match(TokenType.LPAR)) {	10✔
603	const expr = this.test();	10✔
604	this.consume(TokenType.RPAR, "Expected closing ')'");	10✔
605	return new ExprNS.Grouping(startToken, this.previous(), expr);	10✔
606	}
607
NEW 608	if (this.match(TokenType.LSQB)) {	×
NEW 609	const elements = this.list_expr();	×
NEW 610	return new ExprNS.List(startToken, this.previous(), elements);	×
611	}
NEW 612	const startTokenInvalid = this.peek();	×
NEW 613	this.synchronize();	×
NEW 614	throw new ParserErrors.GenericUnexpectedSyntaxError(	×
615	startToken.line,
616	startToken.col,
617	this.source,
618	startTokenInvalid.indexInSource,
619	);
620	}
621
622	//// INVALID RULES
623	private parse_invalid(_startToken: Token, _endToken: Token) {
624	// @TODO invalid rules
625	}
626	}

source-academy / py-slang / 23096627001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous