30916820724

Committed 04 Aug 2026 02:01PM UTC coverage: 86.062% (+0.04%) from 86.027%

Build # 30916820724

Build Type

Pull #399

github

Committed by

web-flow

Commit Message

Merge 59eea6204 into 64acbb674

Pull Request Pull Request #399: py2js: stop showing a fabricated line number, name the enclosing predefined function

Coverage Stats

4680 of 5860 branches covered (79.86%)

Branch coverage included in aggregate %.

78 of 90 new or added lines in 4 files covered. (86.67%)

76 existing lines in 3 files now uncovered.

10435 of 11703 relevant lines covered (89.17%)

166418.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.3

/src/engines/py2js/stdlibBridge.ts

/**
 * py2js engine — stdlib bridge.
 *
 * The stdlib groups (src/stdlib/misc.ts, math.ts, …) are written against the
 * CSE machine's tagged Value union with `(args, source, command, context)`
 * signatures. This bridge exposes them to py2js by converting the engine's
 * native (unboxed) values to tagged Values at the call boundary and back for
 * the result — so both engines run the *same* builtin implementations, and
 * stdlib semantics can never drift between them. Conformance is pinned by
 * src/tests/stdlib-conformance-py2js.test.ts.
 *
 * What crosses the boundary at chapters 1-2: int/float/bool/str/None/complex
 * round-trip losslessly (complex is a PyComplexNumber on both sides); a pair
 * (a 2-element PyList) round-trips to/from CSE's identically-shaped 2-element
 * list Value — chapter 2 has no list-literal syntax and no list.ts group, so
 * pair()/llist() (the only way to construct one) always produce exactly that
 * shape. A function crosses one way *as a value a builtin inspects* (a user function
 * becomes a minimal FunctionValue, so is_function answers True and error
 * messages say 'function'; a py2js builtin becomes a stub BuiltinValue —
 * neither is callable from stdlib code, which no chapter-1/2 builtin
 * attempts) — but it can still flow back out *unchanged*, e.g.
 * `head(pair(1, f))` or `llist(f)`, since no chapter-1/2 builtin fabricates a
 * genuinely new function value, only passes an argument through structurally
 * (into/out of a pair). `functionOrigin` (a WeakMap from the synthetic tagged
 * stand-in back to the real PyFunction) is what makes that round-trip
 * lossless without reconstructing a function from its printable stand-in,
 * which is impossible in general.
 *
 * Error handling: stdlib builtins raise through handleRuntimeError, which
 * records on the bridge's Context and throws the error class — the throw
 * propagates out of the compiled py2js program and is wrapped into a
 * Py2JsRunError by index.ts, preserving the error-class name. The `command`
 * node each builtin receives is a synthetic Call whose callee token carries
 * the builtin's name, so messages that name the callee ("unsupported argument
 * type for math_sin…") stay accurate; source positions point at the program
 * start until py2js grows real error locations.
 *
 * Async builtins (print, input — the stream-based ones) are NOT bridged:
 * their sync py2js replacements live in runtime.ts's native core, and the
 * bridge's Promise guard below is the backstop for any future async stdlib
 * addition. `arity` is also native (py2js functions are not CSE closures);
 * bridged builtins carry pyMinArgs so it reports the same numbers the CSE
 * machine does.
 */
import type { BaseDataVisualizerRunnerPlugin } from "@sourceacademy/runner-data-visualizer";

import { ExprNS } from "../../ast-types";
import { RuntimeSourceError } from "../../errors";
import { Token, TokenType } from "../../tokenizer";
import { GroupName } from "../../stdlib/utils";
import type { Group } from "../../stdlib/utils";
import { Context } from "../cse/context";
import type { Environment } from "../cse/environment";
import type { BuiltinValue, ListValue, Value } from "../cse/stash";
import {
  isPairShaped,
  Py2JsRuntime,
  Py2JsRuntimeError,
  PyFunction,
  PyList,
  PyOpaque,
  pyTypeName,
  PyValue,
} from "./runtime";

function syntheticCallNode(name: string): ExprNS.Call {
  const token = new Token(TokenType.NAME, name, 1, 0, 0);
  token.synthetic = true;
  const callee = new ExprNS.Variable(token, token, token);
  return new ExprNS.Call(token, token, callee, []);
}

/** Maps a synthetic tagged stand-in (built below) back to the real PyFunction
 * it stands in for — see the file header's note on lossless pass-through. */
const functionOrigin = new WeakMap<object, PyFunction>();

/**
 * `memo` maps a PyList already being converted to its (still being filled
 * in) tagged ListValue, keyed by identity and populated *before* recursing
 * into the list's elements/spine — chapter 3's set_head/set_tail, or a
 * literal-list subscript assignment (`a[0] = a`), can build a genuinely
 * cyclic PyList (issue #341), and without this a bridged builtin's argument
 * conversion recurses/loops forever converting it. Populating the memo
 * before descending means a PyList reachable from itself converts to a
 * ListValue reachable from itself, so whatever CSE builtin receives it sees
 * the same kind of cyclic Value graph its own native (non-bridged) callers
 * already have to cope with (e.g. stringify.ts's ancestor tracking) —
 * cycle-handling stays the shared builtin's job, not this bridge's.
 */
function toTagged(v: PyValue, memo: Map<PyList, Value> = new Map()): Value {
  switch (typeof v) {
    case "bigint":
      return { type: "bigint", value: v };
    case "number":
      return { type: "number", value: v };
    case "boolean":
      return { type: "bool", value: v };
    case "string":
      return { type: "string", value: v };
    case "function": {
      // See file header: functions cross as inspectable, non-callable
      // stand-ins. The fallbacks cover bare JS functions that bypassed
      // annotateHostFunction (index.ts establishes the metadata invariant
      // for extraBuiltins; Function#name can be "", hence || not ??).
      const name = v.pyName ?? (v.name || "(anonymous)");
      const tagged: Value = v.pyBuiltin
        ? {
            type: "builtin",
            name,
            minArgs: v.pyMinArgs ?? Math.max(0, v.pyArity ?? v.length),
            func: () => {
              throw new Py2JsRuntimeError(
                "SystemError",
                `stdlib bridge: ${name} cannot be called from a bridged builtin`,
              );
            },
          }
        : {
            type: "function",
            name,
            params: [],
            body: [],
            env: undefined as unknown as Environment,
          };
      functionOrigin.set(tagged, v);
      return tagged;
    }
    default:
      if (v === null) return { type: "none" };
      // CSE has no separate representation for a pair vs. an arbitrary-
      // length list (both are its flat `{type:"list", value: Value[]}` —
      // see src/engines/cse/stash.ts), so converting element-wise here
      // reproduces CSE's own is_list/list_length answers exactly, including
      // on a 2-element list (which CSE cannot tell apart from a pair
      // either — see runtime.ts's PyList doc comment).
      if (Array.isArray(v)) return toTaggedList(v, memo);
      // No chapter-1/2 stdlib builtin accepts an opaque module value as an
      // argument (abs/math_sqrt/etc. all type-check against "opaque" being
      // absent from their accepted types), so this just needs to produce
      // *some* CSE Value the builtin's own dispatch will reject cleanly —
      // matching CSE's own opaque conversion shape (modules.ts).
      if (v instanceof PyOpaque) return { type: "opaque", value: v.typed };
      return { type: "complex", value: v };
  }
}

/**
 * Converts a PyList to CSE's nested list Value. A long proper list built via
 * pair()/llist() (or one built via chapter-3+ mutation into the same shape)
 * has its entire length along the "spine" (chained tail positions), so this
 * walks that spine iteratively rather than recursing — a naive
 * `{ type: "list", value: [toTagged(v[0]), toTagged(v[1])] }` applied
 * per-element via plain recursion would cost one JS stack frame per element
 * just to convert a single argument for one bridged call, and a list of a
 * few thousand elements already overflows the stack, regardless of how
 * tail-recursive the user's own Python is (its own tail calls go through
 * py2js's trampoline; this bridge conversion does not). The walk continues
 * exactly as long as the current node is itself a 2-element list (a chain
 * link) not already seen this walk; it stops — falling through to the final
 * tail conversion — the moment that shape breaks (a flat N-element (N≠2)
 * literal list: zero iterations; a pair whose tail isn't itself a pair: one
 * iteration) or the chain loops back on itself (chapter 3's set_tail, or an
 * N-element list looping back into itself, e.g. `a[0] = a`). `heads[i]`/the
 * final tail are still converted via the ordinary (recursive) toTagged,
 * since each head is normally a scalar leaf; a list-of-lists nested
 * arbitrarily deep through a head position remains a (much rarer)
 * recursion, same as before.
 *
 * Each spine node gets its ListValue placeholder allocated (and memoized)
 * before its tail is known, exactly like toTagged's memo contract — so
 * closing a cycle back onto an earlier node just reuses that node's
 * placeholder as the tail, producing a genuinely cyclic Value graph instead
 * of looping forever building `heads`.
 */
function toTaggedList(v: PyList, memo: Map<PyList, Value>): Value {
  const memoized = memo.get(v);
  if (memoized !== undefined) return memoized;
  if (!isPairShaped(v)) {
    const listValue: ListValue = { type: "list", value: [] };
    memo.set(v, listValue);
    listValue.value = v.map(elem => toTagged(elem, memo));
    return listValue;
  }
  const nodes: PyList[] = [];
  const placeholders: ListValue[] = [];
  let current: PyValue = v;
  while (isPairShaped(current)) {
    const seen = memo.get(current);
    if (seen !== undefined) break;
    const placeholder: ListValue = { type: "list", value: [] };
    memo.set(current, placeholder);
    nodes.push(current);
    placeholders.push(placeholder);
    current = current[1];
  }
  let tail = isPairShaped(current) ? memo.get(current)! : toTagged(current, memo);
  for (let i = nodes.length - 1; i >= 0; i--) {
    placeholders[i].value = [toTagged(nodes[i][0], memo), tail];
    tail = placeholders[i];
  }
  return memo.get(v)!;
}

function fromTagged(name: string, v: Value): PyValue {
  switch (v.type) {
    case "bigint":
    case "number":
    case "string":
    case "complex":
      return v.value;
    case "bool":
      return v.value;
    case "none":
      return null;
    case "list": {
      // Mirrors toTaggedList's iterative spine-walk, in the opposite
      // direction: a long proper list/pair chain a bridged builtin returns
      // (enum_llist, reverse, map, …) is nested 2-element CSE list Values
      // all the way down, so reconstructing it via plain per-element
      // recursion (fromTagged on each tail) would cost one JS stack frame
      // per element — the same failure mode toTaggedList's own doc comment
      // describes, just crossing the boundary in the other direction.
      if (v.value.length !== 2) return v.value.map(el => fromTagged(name, el));
      const heads: PyValue[] = [];
      let current: Value = v;
      while (current.type === "list" && current.value.length === 2) {
        heads.push(fromTagged(name, current.value[0]));
        current = current.value[1];
      }
      let tail = fromTagged(name, current);
      for (let i = heads.length - 1; i >= 0; i--) {
        tail = [heads[i], tail];
      }
      return tail;
    }
    case "function":
    case "builtin": {
      // A function value only ever flows back out as one of THIS bridge's
      // own synthetic stand-ins passed through unchanged (see file header;
      // e.g. head(pair(1, f)) or llist(f)) — never a genuinely new closure a
      // builtin fabricated, which no chapter-1/2 builtin does. Recover the
      // original PyFunction rather than trying to reconstruct one.
      const original = functionOrigin.get(v);
      if (original !== undefined) return original;
      throw new Py2JsRuntimeError(
        "SystemError",
        `stdlib bridge: ${name}() returned a function value the bridge did not itself produce`,
      );
    }
    default:
      // No chapter-1/2 stdlib builtin returns a closure or list-family value
      // other than a pair; reaching this means the bridge needs extending,
      // not that user code is wrong.
      throw new Py2JsRuntimeError(
        "SystemError",
        `stdlib bridge: ${name}() returned an unbridgeable '${v.type}' value`,
      );
  }
}

function bridgeBuiltin(
  rt: Py2JsRuntime,
  name: string,
  builtin: BuiltinValue,
  context: Context,
  source: string,
): PyFunction {
  const command = syntheticCallNode(name);
  const call = builtin.func as (
    args: Value[],
    source: string,
    command: ExprNS.Call,
    context: Context,
  ) => Value | undefined | Promise<Value | undefined>;
  // pyArity -1: argument-count validation is the builtin's own @Validate
  // wrapper, so arity errors carry the CSE machine's exact messages.
  const f = rt.def(name, -1, (...args: PyValue[]) => {
    let result: Value | undefined | Promise<Value | undefined>;
    try {
      result = call(
        args.map(a => toTagged(a)),
        source,
        command,
        context,
      );
    } catch (e) {
      // handleRuntimeError (src/engines/cse/error.ts) throws a
      // RuntimeSourceError — a plain object implementing SourceError, not
      // `extends Error` (see errors.ts) — so it fails `instanceof Error`
      // everywhere up the call chain (including index.ts's own catch
      // blocks), collapsing to the useless "[object Object]" (py-slang#295)
      // instead of surfacing whatever real message .message holds. Convert
      // it into a proper Py2JsRuntimeError here, at the boundary where the
      // CSE-shaped error actually originates, the same way every other
      // py2js-native error already carries its kind as `.name`.
      // error.constructor.name is the exact class name (TypeError,
      // IndexError, ZeroDivisionError, ...) for every RuntimeSourceError
      // subclass — none of them set `.name` themselves (CSE's own
      // displayError falls back to a generic "Error" name for the same
      // reason), so the constructor is the only reliable source for it.
      if (e instanceof RuntimeSourceError) {
        // The builtin itself is named in e.message already ("...for tail: ...");
        // this instead names the *enclosing* predefined function, if any — e.g. a
        // student calling map() never wrote or sees _map, the internal helper that
        // actually calls tail() (py-slang#397).
        const enclosing = rt.enclosingPreludeFunction();
        const message = enclosing
          ? `${e.message} (in predefined function '${enclosing}')`
          : e.message;
        throw new Py2JsRuntimeError(e.constructor.name, message);
      }
      throw e;
    }
    if (result instanceof Promise) {
      throw new Py2JsRuntimeError(
        "RuntimeError",
        `${name}() is asynchronous and not yet supported by the py2js engine`,
      );
    }
    return result === undefined ? null : fromTagged(name, result);
  });
  f.pyBuiltin = true;
  f.pyNative = true;
  f.pyMinArgs = builtin.minArgs;
  return f;
}

/**
 * set_head/set_tail (chapter 3's pair-mutators group): mutate an existing
 * 2-element list *in place*. This cannot go through the generic toTagged/
 * fromTagged round-trip like every other bridged builtin — that round-trip
 * converts the argument into a *fresh* CSE-side value, so a mutation the CSE
 * builtin performs on it (as pairmutator.ts does) would be silently lost
 * rather than visible on the original PyList the caller's Python variable
 * still points to.
 */
function nativeSetPairSlot(name: string, index: 0 | 1, sayPair: boolean): PyFunction {
  const f = ((...args: PyValue[]) => {
    const target = args[0];
    const value = args[1];
    if (isPairShaped(target)) {
      target[index] = value;
      return null;
    }
    throw new Py2JsRuntimeError(
      "TypeError",
      `${name}() expects a pair as first argument, got '${pyTypeName(target, sayPair)}'`,
    );
  }) as PyFunction;
  f.pyName = name;
  f.pyArity = 2;
  f.pyBuiltin = true;
  f.pyNative = true;
  f.pyMinArgs = 2;
  return f;
}

/**
 * stream() (chapter 3's stream group): the one native primitive the group
 * needs — every other stream function (stream_map, stream_filter, …) is pure
 * Python in stream.prelude.ts, already runnable once pairs/closures work, so
 * it never touches this. Not bridged generically because CSE's own
 * StreamBuiltins.stream fabricates a brand new closure (the lazy tail thunk)
 * on every call — the bridge's toTagged/fromTagged round-trip only handles
 * function values that either originated in py2js or pass through
 * unchanged, not ones a CSE builtin invents on the spot — so it's
 * reimplemented directly against py2js's own PyList/PyFunction instead,
 * mirroring StreamBuiltins.stream's recursion exactly.
 *
 * `build` takes an index rather than re-slicing `args` on every lazy step:
 * `args.slice(1)` would copy the remaining N-1 elements at each of N steps,
 * O(N^2) time and space for an N-element stream.
 */
function nativeStream(rt: Py2JsRuntime): PyFunction {
  const build = (args: PyValue[], index: number): PyValue => {
    if (index >= args.length) return null;
    const tail = rt.def("anonymous stream", 0, () => build(args, index + 1));
    tail.pyBuiltin = true;
    tail.pyNative = true;
    return [args[index], tail];
  };
  const f = ((...args: PyValue[]) => build(args, 0)) as PyFunction;
  f.pyName = "stream";
  f.pyArity = -1;
  f.pyBuiltin = true;
  f.pyNative = true;
  f.pyMinArgs = 0;
  return f;
}

/**
 * apply_in_underlying_python(f, xs) (chapter 4's parser/"mce" group): calls
 * `f` with the arguments in linked-list `xs`. CSE's own implementation
 * (ParserBuiltins.apply_in_underlying_python in src/stdlib/parser.ts) pushes
 * onto its own control/stash for the CSE step loop to pick up later, rather
 * than returning a value — semantically incompatible with bridgeBuiltin's
 * generic path, which expects a builtin to return a Value synchronously (a
 * generic bridge attempt would silently no-op: it would run against a fresh,
 * throwaway Context whose control/stash nothing ever steps). Reimplemented
 * natively instead: walk the argument list exactly as permissively as CSE
 * does (any 2-element-list-shaped chain, terminated by anything that isn't,
 * not strictly requiring a None tail) and invoke `f` through the runtime's
 * own synchronous call trampoline (the same
 * one a TS module uses to call back into Python — see Py2JsRuntime.callSync).
 *
 * `parse`/`tokenize` need no such native treatment: both just transform a
 * string into CSE's tagged parse tree (transform() in src/stdlib/parser.ts),
 * built entirely out of 2-element cons cells, which the existing
 * toTagged/fromTagged round-trip already reconstructs correctly as nested
 * PyLists — so they go through the ordinary generic bridge above unchanged.
 */
function walkArgList(xs: PyValue): PyValue[] {
  const args: PyValue[] = [];
  // Cycle guard: set_tail/set_head (chapter 3's pair mutators) can build a
  // genuinely self-referential pair (`p = pair(1, 2); set_tail(p, p)`).
  // Without this, a circular argument list would spin forever, growing
  // `args` without bound until the process runs out of memory — visited
  // tracks list *nodes* by identity (not values), so only an actual cycle
  // back to a node already on this walk trips it, not e.g. two
  // separately-built pairs that happen to compare equal.
  const visited = new Set<object>();
  let current = xs;
  while (isPairShaped(current)) {
    if (visited.has(current)) {
      throw new Py2JsRuntimeError("RuntimeError", "circular list structure in arguments");
    }
    visited.add(current);
    args.push(current[0]);
    current = current[1];
  }
  return args;
}

function nativeApplyInUnderlyingPython(rt: Py2JsRuntime): PyFunction {
  const f = ((...args: PyValue[]) => rt.callSync(args[0], walkArgList(args[1]))) as PyFunction;
  f.pyName = "apply_in_underlying_python";
  f.pyArity = 2;
  f.pyBuiltin = true;
  f.pyNative = true;
  f.pyMinArgs = 2;
  return f;
}

/**
 * draw_data(value1, *values) (bridged as part of the LINKED_LISTS group, alongside
 * pair/llist/head/tail — available from chapter 2 onward, matching the documented signature
 * (docs/lib/linked_list.py) and CSE's identical chapter gating). Reimplemented natively rather than
 * through the generic bridge for two reasons:
 *
 *  - toTagged/toTaggedList (above) walk a pair/list's spine with no cycle guard — fine for every
 *    other bridged builtin, none of which is ever handed a genuinely self-referential structure in
 *    practice, but chapter 3+'s set_head/set_tail can build exactly that, and a user visualizing such
 *    a structure is precisely the case draw_data exists to handle gracefully (a "ref" node, not a
 *    hang). Passing the native args straight to the plugin's sendDrawing — which walks them via
 *    toDataVisualizerNodePy2Js (conductor/dataVisualizer/), with its own refs-based cycle guard —
 *    avoids that conversion entirely.
 *  - sendDrawing is synchronous and fire-and-forget (a channel send), so there is no async result to
 *    thread back through the generic bridge's Value round-trip in the first place.
 *
 * `plugin` is undefined below chapter 2 (bridgeStdlibGroups is never called with one there — see
 * Py2JsEvaluator.ts) and in every standalone/test run (runCodePy2Js et al. pass no dataVisualizer
 * option), in which case this is a silent no-op, exactly like context.dataVisualizer?.sendDrawing(...)
 * on the CSE side when no host conductor is attached.
 */
function nativeDrawData(plugin: BaseDataVisualizerRunnerPlugin<PyValue> | undefined): PyFunction {
  const f = ((...args: PyValue[]) => {
    if (args.length < 1) {
      throw new Py2JsRuntimeError(
        "TypeError",
        `draw_data() takes at least 1 argument (${args.length} given)`,
      );
    }
    plugin?.sendDrawing(args);
    return null;
  }) as PyFunction;
  f.pyName = "draw_data";
  f.pyArity = -1;
  f.pyBuiltin = true;
  f.pyNative = true;
  f.pyMinArgs = 1;
  return f;
}

/**
 * Bridge every builtin (and constant) of the given stdlib groups into py2js
 * native values. `source` is the program text, used by stdlib error
 * constructors for their (currently synthetic) location info.
 */
export function bridgeStdlibGroups(
  rt: Py2JsRuntime,
  groups: Group[],
  source: string,
  variant: number,
  dataVisualizer?: BaseDataVisualizerRunnerPlugin<PyValue>,
): Record<string, PyValue> {
  const context = new Context();
  context.variant = variant;
  const out: Record<string, PyValue> = {};
  for (const group of groups) {
    for (const [name, value] of group.builtins) {
      out[name] =
        value.type === "builtin"
          ? bridgeBuiltin(rt, name, value, context, source)
          : fromTagged(name, value);
    }
    // See nativeSetPairSlot/nativeStream/nativeDrawData's doc comments for why these groups'
    // primitives are reimplemented natively instead of left as the generic bridge produced above.
    if (group.name === GroupName.PAIRMUTATORS) {
      out.set_head = nativeSetPairSlot("set_head", 0, variant <= 2);
      out.set_tail = nativeSetPairSlot("set_tail", 1, variant <= 2);
    }
    if (group.name === GroupName.STREAMS) {
      out.stream = nativeStream(rt);
    }
    if (group.name === GroupName.MCE) {
      out.apply_in_underlying_python = nativeApplyInUnderlyingPython(rt);
    }
    // LINKED_LISTS (pair/llist/head/tail) is chapter 2's own "list library" — draw_data belongs
    // alongside it rather than in its own group (unlike the CSE machine, which uses a dedicated
    // DATA_VISUALIZER group), so it inherits the exact same chapter-2-onward availability.
    if (group.name === GroupName.LINKED_LISTS) {
      out.draw_data = nativeDrawData(dataVisualizer);
    }
  }
  return out;
}

source-academy / py-slang / 30916820724

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous