23716341471

Committed 29 Mar 2026 06:45PM UTC coverage: 40.723% (-0.5%) from 41.233%

Build # 23716341471

Build Type

Pull #124

github

Committed by

web-flow

Commit Message

Merge 5b65dfe02 into 7b1e59a17

Pull Request Pull Request #124: Fix types of Python grammar

Coverage Stats

198 of 884 branches covered (22.4%)

Branch coverage included in aggregate %.

54 of 55 new or added lines in 2 files covered. (98.18%)

3 existing lines in 2 files now uncovered.

1222 of 2603 relevant lines covered (46.95%)

51.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.81

/src/parser/lexer.ts

/**
 * Two-pass Python lexer: Moo tokenization → indent/dedent injection.
 *
 * Pass 1: moo.compile() produces a flat token stream.
 * Pass 2: processTokens() strips whitespace/comments, tracks enclosure
 *         depth, and emits synthetic indent/dedent tokens.
 */

import moo from "moo";
import { UnexpectedIndentError, InconsistentDedentError } from "./lexer-errors";

// ── Moo configuration (unchanged) ──────────────────────────────────────────

const kwType = moo.keywords({
  kw_def: "def",
  kw_if: "if",
  kw_elif: "elif",
  kw_else: "else",
  kw_while: "while",
  kw_for: "for",
  kw_in: "in",
  kw_return: "return",
  kw_pass: "pass",
  kw_break: "break",
  kw_continue: "continue",
  kw_and: "and",
  kw_or: "or",
  kw_not: "not",
  kw_is: "is",
  kw_lambda: "lambda",
  kw_from: "from",
  kw_import: "import",
  kw_global: "global",
  kw_nonlocal: "nonlocal",
  kw_as: "as",
  kw_assert: "assert",
  kw_True: "True",
  kw_False: "False",
  kw_None: "None",
  // Forbidden keywords (surface as their own type so callers can error nicely)
  forbidden_async: "async",
  forbidden_await: "await",
  forbidden_yield: "yield",
  forbidden_with: "with",
  forbidden_del: "del",
  forbidden_try: "try",
  forbidden_except: "except",
  forbidden_finally: "finally",
  forbidden_raise: "raise",
  forbidden_class: "class",
});

const mooLexer = moo.compile({
  newline: { match: /\n/, lineBreaks: true },
  ws: /[ \t]+/,
  comment: /#[^\n]*/,

  number_complex: /(?:\d+\.?\d*|\.\d+)[jJ]/,
  number_float: /(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?/,
  number_hex: /0[xX][0-9a-fA-F]+/,
  number_oct: /0[oO][0-7]+/,
  number_bin: /0[bB][01]+/,
  number_int: /\d+/,

  string_triple_double: /"""(?:[^\\]|\\.)*?"""/,
  string_triple_single: /'''(?:[^\\]|\\.)*?'''/,
  string_double: /"(?:[^"\\]|\\.)*"/,
  string_single: /'(?:[^'\\]|\\.)*'/,

  doublestar: "**",
  doubleslash: "//",
  doubleequal: "==",
  notequal: "!=",
  lessequal: "<=",
  greaterequal: ">=",
  doublecolon: "::",
  ellipsis: "...",

  lparen: "(",
  rparen: ")",
  lsqb: "[",
  rsqb: "]",
  lbrace: "{",
  rbrace: "}",
  colon: ":",
  comma: ",",
  plus: "+",
  minus: "-",
  star: "*",
  slash: "/",
  percent: "%",
  less: "<",
  greater: ">",
  equal: "=",
  dot: ".",
  semi: ";",

  name: { match: /[a-zA-Z_][a-zA-Z0-9_]*/, type: kwType },
});

// ── Openers / closers for enclosure tracking ───────────────────────────────

const OPENERS = new Set(["(", "[", "{"]);
const CLOSERS = new Set([")", "]", "}"]);

// ── Synthetic token factory ────────────────────────────────────────────────

function syntheticToken(type: string, ref: moo.Token): moo.Token {
  return {
    type,
    value: "",
    text: "",
    toString: ref.toString,
    offset: ref.offset,
    lineBreaks: 0,
    line: ref.line,
    col: ref.col,
  };
}

// ── Pass 2: processTokens ──────────────────────────────────────────────────

function processTokens(raw: moo.Token[]): moo.Token[] {
  const out: moo.Token[] = [];
  const indentStack: string[] = [""];
  let enclosureDepth = 0;
  let i = 0;

  // Reject leading indentation (whitespace before the first real token
  // with no preceding newline).
  {
    let j = 0;
    while (j < raw.length && (raw[j].type === "comment" || raw[j].type === "newline")) j++;
    if (j < raw.length && raw[j].type === "ws") {
      throw new UnexpectedIndentError(raw[j].line, raw[j].col);
    }
  }

  while (i < raw.length) {
    const tok = raw[i];

    // Always skip whitespace and comments
    if (tok.type === "ws" || tok.type === "comment") {
      i++;
      continue;
    }

    // Track enclosure depth
    if (OPENERS.has(tok.text)) {
      enclosureDepth++;
      out.push(tok);
      i++;
      continue;
    }
    if (CLOSERS.has(tok.text)) {
      enclosureDepth--;
      out.push(tok);
      i++;
      continue;
    }

    // Inside enclosures: skip newlines
    if (tok.type === "newline" && enclosureDepth > 0) {
      i++;
      continue;
    }

    // Newline outside enclosures: emit newline then handle indentation
    if (tok.type === "newline") {
      out.push(tok);
      i++;

      // Consume blank lines, comments, and whitespace to find the next
      // real token's indentation level.
      let indent = "";
      while (i < raw.length) {
        const next = raw[i];
        if (next.type === "ws") {
          indent = next.text;
          i++;
          continue;
        }
        if (next.type === "comment") {
          i++;
          // After a comment there must be a newline (or EOF).
          // Skip the newline too, then reset indent for the next line.
          if (i < raw.length && raw[i].type === "newline") {
            i++;
          }
          indent = "";
          continue;
        }
        if (next.type === "newline") {
          // Blank line — skip it, reset indent
          i++;
          indent = "";
          continue;
        }
        // Found a real token
        break;
      }

      // If we've hit EOF after newlines, just emit remaining dedents
      if (i >= raw.length) {
        const ref = raw[raw.length - 1];
        while (indentStack.length > 1) {
          indentStack.pop();
          out.push(syntheticToken("dedent", ref));
        }
        continue;
      }

      const currentIndent = indentStack[indentStack.length - 1];
      if (indent === currentIndent) {
        // Same level — nothing to do
      } else if (indent.startsWith(currentIndent) && indent.length > currentIndent.length) {
        // Deeper indent
        indentStack.push(indent);
        out.push(syntheticToken("indent", raw[i]));
      } else {
        // Dedent — pop until we find a matching level
        while (indentStack.length > 1 && indentStack[indentStack.length - 1] !== indent) {
          indentStack.pop();
          out.push(syntheticToken("dedent", raw[i]));
        }
        if (indentStack[indentStack.length - 1] !== indent) {
          throw new InconsistentDedentError(raw[i].line, raw[i].col);
        }
      }
      continue;
    }

    // Everything else: emit as-is
    out.push(tok);
    i++;
  }

  // EOF: emit remaining dedents
  if (indentStack.length > 1) {
    const ref =
      raw.length > 0
        ? raw[raw.length - 1]
        : ({
            toString: () => "",
            offset: 0,
            line: 1,
            col: 1,
          } as moo.Token);
    while (indentStack.length > 1) {
      indentStack.pop();
      out.push(syntheticToken("dedent", ref));
    }
  }

  return out;
}

// ── PythonLexer (Nearley-compatible wrapper) ───────────────────────────────

interface PythonLexerState extends moo.LexerState {
  pos: number;
}

class PythonLexer implements moo.Lexer {
  private tokens: moo.Token[] = [];
  private pos = 0;

  reset(data?: string, state?: moo.LexerState): this {
    if (state && "pos" in state) {
      this.pos = (state as PythonLexerState).pos;
    } else if (data !== undefined) {
      mooLexer.reset(data);
      const raw: moo.Token[] = [];
      let tok: moo.Token | undefined;
      while ((tok = mooLexer.next())) {
        raw.push(tok);
      }
      this.tokens = processTokens(raw);
      this.pos = 0;
    }
    return this;
  }

  next(): moo.Token | undefined {
    if (this.pos >= this.tokens.length) return undefined;
    return this.tokens[this.pos++];
  }

  save(): moo.LexerState {
    return { pos: this.pos } as unknown as moo.LexerState;
  }

  has(name: string): boolean {
    return name === "indent" || name === "dedent" || mooLexer.has(name);
  }

  formatError(token?: moo.Token, message?: string): string {
    return mooLexer.formatError(token as moo.Token, message);
  }

  pushState(state: string): void {
    mooLexer.pushState(state);
  }

  popState(): void {
    mooLexer.popState();
  }

  setState(state: string): void {
    mooLexer.setState(state);
  }

  [Symbol.iterator](): Iterator<moo.Token> {
    return {
      next: (): IteratorResult<moo.Token> => {
        const token = this.next();
        return { value: token as moo.Token, done: !token };
      },
    };
  }
}
const pythonLexer: moo.Lexer = new PythonLexer();
export default pythonLexer;

1	/**
2	* Two-pass Python lexer: Moo tokenization → indent/dedent injection.
3	*
4	* Pass 1: moo.compile() produces a flat token stream.
5	* Pass 2: processTokens() strips whitespace/comments, tracks enclosure
6	* depth, and emits synthetic indent/dedent tokens.
7	*/
8
9	import moo from "moo";	4✔
10	import { UnexpectedIndentError, InconsistentDedentError } from "./lexer-errors";	4✔
11
12	// ── Moo configuration (unchanged) ──────────────────────────────────────────
13
14	const kwType = moo.keywords({	4✔
15	kw_def: "def",
16	kw_if: "if",
17	kw_elif: "elif",
18	kw_else: "else",
19	kw_while: "while",
20	kw_for: "for",
21	kw_in: "in",
22	kw_return: "return",
23	kw_pass: "pass",
24	kw_break: "break",
25	kw_continue: "continue",
26	kw_and: "and",
27	kw_or: "or",
28	kw_not: "not",
29	kw_is: "is",
30	kw_lambda: "lambda",
31	kw_from: "from",
32	kw_import: "import",
33	kw_global: "global",
34	kw_nonlocal: "nonlocal",
35	kw_as: "as",
36	kw_assert: "assert",
37	kw_True: "True",
38	kw_False: "False",
39	kw_None: "None",
40	// Forbidden keywords (surface as their own type so callers can error nicely)
41	forbidden_async: "async",
42	forbidden_await: "await",
43	forbidden_yield: "yield",
44	forbidden_with: "with",
45	forbidden_del: "del",
46	forbidden_try: "try",
47	forbidden_except: "except",
48	forbidden_finally: "finally",
49	forbidden_raise: "raise",
50	forbidden_class: "class",
51	});
52
53	const mooLexer = moo.compile({	4✔
54	newline: { match: /\n/, lineBreaks: true },
55	ws: /[ \t]+/,
56	comment: /#[^\n]*/,
57
58	number_complex: /(?:\d+\.?\d*\|\.\d+)[jJ]/,
59	number_float: /(?:\d+\.\d*\|\.\d+)(?:[eE][+-]?\d+)?/,
60	number_hex: /0[xX][0-9a-fA-F]+/,
61	number_oct: /0[oO][0-7]+/,
62	number_bin: /0[bB][01]+/,
63	number_int: /\d+/,
64
65	string_triple_double: /"""(?:[^\\]\|\\.)*?"""/,
66	string_triple_single: /'''(?:[^\\]\|\\.)*?'''/,
67	string_double: /"(?:[^"\\]\|\\.)*"/,
68	string_single: /'(?:[^'\\]\|\\.)*'/,
69
70	doublestar: "**",
71	doubleslash: "//",
72	doubleequal: "==",
73	notequal: "!=",
74	lessequal: "<=",
75	greaterequal: ">=",
76	doublecolon: "::",
77	ellipsis: "...",
78
79	lparen: "(",
80	rparen: ")",
81	lsqb: "[",
82	rsqb: "]",
83	lbrace: "{",
84	rbrace: "}",
85	colon: ":",
86	comma: ",",
87	plus: "+",
88	minus: "-",
89	star: "*",
90	slash: "/",
91	percent: "%",
92	less: "<",
93	greater: ">",
94	equal: "=",
95	dot: ".",
96	semi: ";",
97
98	name: { match: /[a-zA-Z_][a-zA-Z0-9_]*/, type: kwType },
99	});
100
101	// ── Openers / closers for enclosure tracking ───────────────────────────────
102
103	const OPENERS = new Set(["(", "[", "{"]);	4✔
104	const CLOSERS = new Set([")", "]", "}"]);	4✔
105
106	// ── Synthetic token factory ────────────────────────────────────────────────
107
108	function syntheticToken(type: string, ref: moo.Token): moo.Token {
109	return {	292✔
110	type,
111	value: "",
112	text: "",
113	toString: ref.toString,
114	offset: ref.offset,
115	lineBreaks: 0,
116	line: ref.line,
117	col: ref.col,
118	};
119	}
120
121	// ── Pass 2: processTokens ──────────────────────────────────────────────────
122
123	function processTokens(raw: moo.Token[]): moo.Token[] {
124	const out: moo.Token[] = [];	287✔
125	const indentStack: string[] = [""];	287✔
126	let enclosureDepth = 0;	287✔
127	let i = 0;	287✔
128
129	// Reject leading indentation (whitespace before the first real token
130	// with no preceding newline).
131	{
132	let j = 0;	287✔
133	while (j < raw.length && (raw[j].type === "comment" \|\| raw[j].type === "newline")) j++;	287✔
134	if (j < raw.length && raw[j].type === "ws") {	287✔
135	throw new UnexpectedIndentError(raw[j].line, raw[j].col);	1✔
136	}
137	}
138
139	while (i < raw.length) {	286✔
140	const tok = raw[i];	3,618✔
141
142	// Always skip whitespace and comments
143	if (tok.type === "ws" \|\| tok.type === "comment") {	3,618✔
144	i++;	888✔
145	continue;	888✔
146	}
147
148	// Track enclosure depth
149	if (OPENERS.has(tok.text)) {	2,730✔
150	enclosureDepth++;	178✔
151	out.push(tok);	178✔
152	i++;	178✔
153	continue;	178✔
154	}
155	if (CLOSERS.has(tok.text)) {	2,552✔
156	enclosureDepth--;	177✔
157	out.push(tok);	177✔
158	i++;	177✔
159	continue;	177✔
160	}
161
162	// Inside enclosures: skip newlines
163	if (tok.type === "newline" && enclosureDepth > 0) {	2,375✔
164	i++;	21✔
165	continue;	21✔
166	}
167
168	// Newline outside enclosures: emit newline then handle indentation
169	if (tok.type === "newline") {	2,354✔
170	out.push(tok);	588✔
171	i++;	588✔
172
173	// Consume blank lines, comments, and whitespace to find the next
174	// real token's indentation level.
175	let indent = "";	588✔
176	while (i < raw.length) {	588✔
177	const next = raw[i];	561✔
178	if (next.type === "ws") {	561✔
179	indent = next.text;	213✔
180	i++;	213✔
181	continue;	213✔
182	}
183	if (next.type === "comment") {	348✔
184	i++;	3✔
185	// After a comment there must be a newline (or EOF).
186	// Skip the newline too, then reset indent for the next line.
187	if (i < raw.length && raw[i].type === "newline") {	3✔
188	i++;	3✔
189	}
190	indent = "";	3✔
191	continue;	3✔
192	}
193	if (next.type === "newline") {	345✔
194	// Blank line — skip it, reset indent
195	i++;	38✔
196	indent = "";	38✔
197	continue;	38✔
198	}
199	// Found a real token
200	break;	307✔
201	}
202
203	// If we've hit EOF after newlines, just emit remaining dedents
204	if (i >= raw.length) {	588✔
205	const ref = raw[raw.length - 1];	281✔
206	while (indentStack.length > 1) {	281✔
207	indentStack.pop();	74✔
208	out.push(syntheticToken("dedent", ref));	74✔
209	}
210	continue;	281✔
211	}
212
213	const currentIndent = indentStack[indentStack.length - 1];	307✔
214	if (indent === currentIndent) {	307✔
215	// Same level — nothing to do
216	} else if (indent.startsWith(currentIndent) && indent.length > currentIndent.length) {	208✔
217	// Deeper indent
218	indentStack.push(indent);	146✔
219	out.push(syntheticToken("indent", raw[i]));	146✔
220	} else {
221	// Dedent — pop until we find a matching level
222	while (indentStack.length > 1 && indentStack[indentStack.length - 1] !== indent) {	62✔
223	indentStack.pop();	72✔
224	out.push(syntheticToken("dedent", raw[i]));	72✔
225	}
226	if (indentStack[indentStack.length - 1] !== indent) {	62✔
227	throw new InconsistentDedentError(raw[i].line, raw[i].col);	2✔
228	}
229	}
230	continue;	305✔
231	}
232
233	// Everything else: emit as-is
234	out.push(tok);	1,766✔
235	i++;	1,766✔
236	}
237
238	// EOF: emit remaining dedents
239	if (indentStack.length > 1) {	284!
240	const ref =
241	raw.length > 0	×
242	? raw[raw.length - 1]
243	: ({
244	toString: () => "",	×
245	offset: 0,
246	line: 1,
247	col: 1,
248	} as moo.Token);
249	while (indentStack.length > 1) {	×
250	indentStack.pop();	×
251	out.push(syntheticToken("dedent", ref));	×
252	}
253	}
254
255	return out;	284✔
256	}
257
258	// ── PythonLexer (Nearley-compatible wrapper) ───────────────────────────────
259
260	interface PythonLexerState extends moo.LexerState {
261	pos: number;
262	}
263
264	class PythonLexer implements moo.Lexer {
265	private tokens: moo.Token[] = [];	4✔
266	private pos = 0;	4✔
267
268	reset(data?: string, state?: moo.LexerState): this {
269	if (state && "pos" in state) {	287!
270	this.pos = (state as PythonLexerState).pos;	×
271	} else if (data !== undefined) {	287✔
272	mooLexer.reset(data);	287✔
273	const raw: moo.Token[] = [];	287✔
274	let tok: moo.Token \| undefined;
275	while ((tok = mooLexer.next())) {	287✔
276	raw.push(tok);	3,894✔
277	}
278	this.tokens = processTokens(raw);	287✔
279	this.pos = 0;	284✔
280	}
281	return this;	284✔
282	}
283
284	next(): moo.Token \| undefined {
285	if (this.pos >= this.tokens.length) return undefined;	3,240✔
286	return this.tokens[this.pos++];	2,960✔
287	}
288
289	save(): moo.LexerState {
290	return { pos: this.pos } as unknown as moo.LexerState;	267✔
291	}
292
293	has(name: string): boolean {
UNCOV 294	return name === "indent" \|\| name === "dedent" \|\| mooLexer.has(name);	×
295	}
296
297	formatError(token?: moo.Token, message?: string): string {
298	return mooLexer.formatError(token as moo.Token, message);	4✔
299	}
300
301	pushState(state: string): void {
302	mooLexer.pushState(state);	×
303	}
304
305	popState(): void {
306	mooLexer.popState();	×
307	}
308
309	setState(state: string): void {
310	mooLexer.setState(state);	×
311	}
312
313	[Symbol.iterator](): Iterator<moo.Token> {
314	return {	×
315	next: (): IteratorResult<moo.Token> => {
316	const token = this.next();	×
317	return { value: token as moo.Token, done: !token };	×
318	},
319	};
320	}
321	}
322	const pythonLexer: moo.Lexer = new PythonLexer();	4✔
323	export default pythonLexer;	4✔

source-academy / py-slang / 23716341471

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous