2055

pending completion

Build # 2055

Build Type

Pull #636

travis-ci-com

Committed by

web-flow

Commit Message

Merge a5567a1dc into c818889a1

Pull Request Pull Request #636: Clean up the template build system

Run Details

9908 of 19994 branches covered (49.55%)

Branch coverage included in aggregate %.

45 of 75 new or added lines in 9 files covered. (60.0%)

1 existing line in 1 file now uncovered.

23415 of 39424 relevant lines covered (59.39%)

497791.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.84

/lib/sentence-generator/compiler/meta_ast.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as path from 'path';
import assert from 'assert';

import { stringEscape } from '../../utils/escaping';
import * as TemplateGrammar from '../../utils/template-string/grammar';
import * as I18n from '../../i18n';

export class NodeVisitor {
    visitImport(stmt : Import) {}

    visitContextStmt(stmt : ContextStmt) {}
    visitNonTerminalStmt(stmt : NonTerminalStmt) {}
    visitKeyFunctionDeclaration(stmt : KeyFunctionDeclarationStmt) {}

    visitOldStyleExpansionRule(stmt : OldStyleExpansion) {}
    visitNewStyleExpansionRule(stmt : NewStyleExpansion) {}
    visitConstantsRule(stmt : Constants) {}
    visitConditionRule(stmt : Condition) {}

    visitNonTerminalRuleHead(node : RuleHeadPart) {}
}

export class Grammar {
    constructor(public comment : string,
                public statements : Statement[]) {
    }

    visit(visitor : NodeVisitor) {
        for (const stmt of this.statements)
            stmt.visit(visitor);
    }

    codegen(filename : string) : string {
        let buffer = '';

        buffer += (this.comment);
        for (const stmt of this.statements) {
            if (stmt instanceof JSImportStmt)
                buffer += stmt.codegen();
        }

        let runtimepath = path.relative(path.dirname(filename), require.resolve('../runtime'));
        if (runtimepath.endsWith('.ts'))
            runtimepath = runtimepath.substring(0, runtimepath.length-3);
        let generatorpath = path.relative(path.dirname(filename), require.resolve('../generator'));
        if (generatorpath.endsWith('.ts'))
            generatorpath = generatorpath.substring(0, generatorpath.length-3);
        let typespath = path.relative(path.dirname(filename), require.resolve('../types'));
        if (typespath.endsWith('.ts'))
            typespath = typespath.substring(0, typespath.length-3);
        let i18npath = path.relative(path.dirname(filename), require.resolve('../../i18n'));
        if (i18npath.endsWith('.ts'))
            i18npath = i18npath.substring(0, i18npath.length-3);
        buffer += `import * as $runtime from ${stringEscape(runtimepath)};\n`;
        buffer += `import type $SentenceGenerator from ${stringEscape(generatorpath)};\n`;
        buffer += `import * as $SentenceGeneratorTypes from ${stringEscape(typespath)};\n`;
        buffer += `import * as $I18n from ${stringEscape(i18npath)};\n`;
        buffer += `export default async function($options : $SentenceGeneratorTypes.GrammarOptions, $locale : $I18n.LanguagePack, $grammar : $SentenceGenerator<any, any>, $loader : ThingpediaLoader) : Promise<void> {\n`;
        for (const stmt of this.statements) {
            if (stmt instanceof JSImportStmt)
                continue;
            buffer += stmt.codegen('   ');
        }
        buffer += '};\n';

        return buffer;
    }
}

export abstract class Statement {
    static CodeBlock : typeof CodeBlock;
    static JSImportStmt : typeof JSImportStmt;
    static NonTerminal : typeof NonTerminalStmt;
    static Context : typeof ContextStmt;
    static FunctionDeclaration : typeof FunctionDeclarationStmt;
    static ForLoop : typeof ForLoop;
    static If : typeof IfStmt;
    static Import : typeof Import;
    static KeyFunctionDeclaration : typeof KeyFunctionDeclarationStmt;

    abstract codegen(prefix ?: string) : string;
    abstract visit(visitor : NodeVisitor) : void;
}

export class CodeBlock extends Statement {
    constructor(public code : string) {
        super();
    }

    codegen() : string {
        return this.code;
    }

    visit(visitor : NodeVisitor) {}
}
Statement.CodeBlock = CodeBlock;

export class JSImportStmt extends Statement {
    constructor(public code : string) {
        super();
    }

    codegen() : string {
        return `import ${this.code};\n`;
    }

    visit(visitor : NodeVisitor) {}
}
Statement.JSImportStmt = JSImportStmt;

export class KeyFunctionDeclarationStmt extends Statement {
    constructor(public decls : Array<[string, string]>) {
        super();
    }

    codegen() : string {
        return ''; // this is a type declaration processed by the compiler,
                   // it generates no code
    }

    visit(visitor : NodeVisitor) {
        visitor.visitKeyFunctionDeclaration(this);
    }
}
Statement.KeyFunctionDeclaration = KeyFunctionDeclarationStmt;

export class NonTerminalStmt extends Statement {
    keyfn = 'undefined';

    constructor(public name : string,
                public type : string|undefined,
                public rules : Rule[]) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitNonTerminalStmt(this);
        for (const rule of this.rules)
            rule.visit(visitor);
    }

    codegen(prefix = '') : string {
        let buffer = '';
        buffer += (`${prefix}$grammar.declareSymbol(${stringEscape(this.name)});\n`);
        for (const rule of this.rules)
            buffer += rule.codegen(this.name, prefix, this.type || 'any', this.keyfn);
        return buffer;
    }
}
Statement.NonTerminal = NonTerminalStmt;

export class ContextStmt extends Statement {
    constructor(public names : string[],
                public type : string|undefined) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitContextStmt(this);
    }

    codegen(prefix = '') : string {
        return this.names.map((name) => `${prefix}$grammar.declareContext(${stringEscape(name)});\n`).join('');
    }
}
Statement.Context = ContextStmt;

export class FunctionDeclarationStmt extends Statement {
    constructor(public name : string,
                public args : string,
                public code : string,
                public type = 'any') {
        super();
    }

    visit(visitor : NodeVisitor) {}

    codegen(prefix = '') : string {
        return `${prefix}$grammar.declareFunction('${this.name}', (${this.args}) : ${this.type} => {${this.code}});\n`;
    }
}
Statement.FunctionDeclaration = FunctionDeclarationStmt;

export class ForLoop extends Statement {
    constructor(public head : string,
                public statements : Statement[]) {
        super();
    }

    visit(visitor : NodeVisitor) {
        for (const stmt of this.statements)
            stmt.visit(visitor);
    }

    codegen(prefix = '') : string {
        let buffer = '';
        buffer += (`${prefix}for (${this.head}) {\n`);
        for (const stmt of this.statements)
            buffer += stmt.codegen(prefix + '    ');
        buffer += (`${prefix}}\n`);
        return buffer;
    }
}
Statement.ForLoop = ForLoop;

export class IfStmt extends Statement {
    constructor(public cond : string,
                public iftrue : Statement[],
                public iffalse : Statement[]) {
        super();
    }

    visit(visitor : NodeVisitor) {
        for (const stmt of this.iftrue)
            stmt.visit(visitor);
        for (const stmt of this.iffalse)
            stmt.visit(visitor);
    }

    codegen(prefix = '') : string {
        let buffer = '';
        buffer += (`${prefix}if (${this.cond}) {\n`);
        for (const stmt of this.iftrue)
            buffer += stmt.codegen(prefix + '    ');
        if (this.iffalse.length > 0) {
            buffer += (`${prefix}} else {\n`);
            for (const stmt of this.iffalse)
                buffer += stmt.codegen(prefix + '    ');
        }
        buffer += (`${prefix}}\n`);
        return buffer;
    }
}
Statement.If = IfStmt;

export class Import extends Statement {
    constructor(public what : string) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitImport(this);
    }

    codegen(prefix = '') : string {
        return `${prefix}await (await $runtime.import(${stringEscape(this.what)}, __dirname))($options, $locale, $grammar, $loader);\n`;
    }
}
Statement.Import = Import;

interface RuleAttribute {
    name : string;
    code : string;
}

export class RuleAttributes {
    constructor(public attributes : RuleAttribute[] = []) {
    }

    codegen() : string {
        let buf = '{ ';
        buf += this.attributes.map((attr) => `${attr.name}: (${attr.code})`).join(', ');
        buf += ' }';
        return buf;
    }
}

export abstract class Rule {
    static Constants : typeof Constants;
    static Condition : typeof Condition;
    static OldStyleExpansion : typeof OldStyleExpansion;
    static NewStyleExpansion : typeof NewStyleExpansion;

    abstract codegen(nonTerminal : string, prefix : string, type : string, keyfn : string) : string;
    abstract visit(visitor : NodeVisitor) : void;
}

export class Constants extends Rule {
    constructor(public token : string,
                public typeCode : string,
                public attrs : RuleAttributes) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitConstantsRule(this);
    }

    codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {
        return `${prefix}$grammar.addConstants(${stringEscape(nonTerminal)}, ${stringEscape(this.token)}, ${this.typeCode}, ${keyfn}, ${this.attrs.codegen()});\n`;
    }
}
Rule.Constants = Constants;

function makeBodyLambda(head : NonTerminalRuleHead[],
                        body : string,
                        type = 'any') : string {
    const bodyArgs : string[] = [];
    let i = 0;
    for (const headPart of head) {
        if (headPart instanceof NonTerminalRuleHead && headPart.name)
            bodyArgs.push(headPart.name + ' : ' + headPart.type);
        else
            bodyArgs.push(`$${i++}` + ' : ' + headPart.type);
    }

    return `(${bodyArgs.join(', ')}) : (${type})|null => ${body}`;
}

export class OldStyleExpansion extends Rule {
    constructor(public head : RuleHeadPart[],
                public bodyCode : string,
                public attrs : RuleAttributes) {
        super();
        assert(Array.isArray(head));
    }

    visit(visitor : NodeVisitor) {
        visitor.visitOldStyleExpansionRule(this);
        for (const head of this.head)
            head.visit(visitor);
    }

    codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {
        const nonTerminalChildren : NonTerminalRuleHead[] = this.head.filter((h) : h is NonTerminalRuleHead => h instanceof NonTerminalRuleHead);
        const expanderCode = makeBodyLambda(nonTerminalChildren, this.bodyCode, type);

        let template = '"' + this.head.map((h) => h.getTemplate()).join(' ') + '"';

        // generate code to lookup the translation of the template if meaningful
        // (skip if this template has only one component)
        if (this.head.length > 1 && template !== '""')
            template = `$locale._(${template})`;

        return `${prefix}$grammar.addRule(${stringEscape(nonTerminal)}, [${nonTerminalChildren.map((h, i) => h.codegen(nonTerminalChildren, i)).join(', ')}], ${template}, (${expanderCode}), ${keyfn}, ${this.attrs.codegen()});\n`;
    }
}
Rule.OldStyleExpansion = OldStyleExpansion;

export class NewStyleExpansion extends Rule {
    constructor(public nonTerminals : NonTerminalRuleHead[],
                public sentenceTemplate : string,
                public bodyCode : string,
                public attrs : RuleAttributes) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitNewStyleExpansionRule(this);
        for (const nt of this.nonTerminals)
            nt.visit(visitor);
    }

    codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {
        const expanderCode = makeBodyLambda(this.nonTerminals, this.bodyCode, type);

        // try parsing the template and preprocessing, so we catch errors eagerly
        try {
            TemplateGrammar.parse(this.sentenceTemplate).preprocess(I18n.get('en-US'), this.nonTerminals.map((e) => e.name ?? e.symbol));
        } catch(e) {
            throw new Error(`Failed to parse template string for ${nonTerminal} = ${this.sentenceTemplate} (${this.nonTerminals.join(', ')}): ${e.message}`);
        }

        let gettext;
        if (this.sentenceTemplate)
            gettext = `$locale._(${stringEscape(this.sentenceTemplate)})`;
        else
            gettext = "''";

        return `${prefix}$grammar.addRule(${stringEscape(nonTerminal)}, [${this.nonTerminals.map((h, i) => h.codegen(this.nonTerminals, i)).join(', ')}], ${gettext}, (${expanderCode}), ${keyfn}, ${this.attrs.codegen()});\n`;
    }
}
Rule.NewStyleExpansion = NewStyleExpansion;

export class Condition extends Rule {
    constructor(public flag : string,
                public rules : Rule[]) {
        super();
    }

    visit(visitor : NodeVisitor) {
        visitor.visitConditionRule(this);
        for (const rule of this.rules)
            rule.visit(visitor);
    }

    codegen(nonTerminal : string, prefix : string, type : string, keyfn : string) : string {
        const flag = this.flag.startsWith('?') ?
            `$options.flags.${this.flag.substring(1)}` :
            `!$options.flags.${this.flag.substring(1)}`;

        let buffer = '';
        buffer += (`${prefix}if (${flag}) {\n`);
        for (const rule of this.rules)
            buffer += rule.codegen(nonTerminal, prefix + '    ', type, keyfn);
        buffer += (`${prefix}}\n`);
        return buffer;
    }
}
Rule.Condition = Condition;

export abstract class NonTerminalConstraint {
    static Constant : typeof ConstantNonTerminalConstraint;
    static Equality : typeof EqualityNonTerminalConstraint;

    abstract codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string, ourIndex : number) : string;
}

export class EqualityNonTerminalConstraint extends NonTerminalConstraint {
    constructor(public ourIndexName : string,
                public nonTermRef : string,
                public theirIndexName : string) {
        super();
    }

    codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string, ourIndex : number) {
        if (ourKeyFn === 'undefined')
            console.error(`WARNING: key function is not set in constraint {${this.ourIndexName} = ${this.nonTermRef}.${this.theirIndexName}}, cannot check correctness statically`);
        const ourTypeConstraint = ourKeyFn === 'undefined' ? '' :
            ` as (${stringEscape(this.ourIndexName)} extends keyof ReturnType<typeof ${ourKeyFn}> ? string : void)`;

        let nonTermIndex, theirKeyFn;
        if (/^[0-9]+/.test(this.nonTermRef)) {
            nonTermIndex = parseInt(this.nonTermRef, 10);
            assert(allNonTerminals[nonTermIndex]);

            theirKeyFn = allNonTerminals[nonTermIndex];
        } else {
            for (let i = 0; i < allNonTerminals.length; i++) {
                const part = allNonTerminals[i];
                if (part.name === this.nonTermRef) {
                    nonTermIndex = i;
                    theirKeyFn = part.keyfn;
                    break;
                }
            }
            if (nonTermIndex === undefined)
                throw new Error(`Invalid non-terminal backreference to ${this.nonTermRef} for equality constraint of ${allNonTerminals[ourIndex]} (alias not found)`);
            if (nonTermIndex >= ourIndex)
                throw new Error(`Invalid non-terminal backreference to ${this.nonTermRef} for equality constraint of ${allNonTerminals[ourIndex]} (alias must precede usage)`);
        }
        if (theirKeyFn === 'undefined')
            console.error(`WARNING: key function is not set in constraint {${this.ourIndexName} = ${this.nonTermRef}.${this.theirIndexName}}, cannot check correctness statically`);
        const theirTypeConstraint = theirKeyFn === 'undefined' ? '' :
            ` as (${stringEscape(this.theirIndexName)} extends keyof ReturnType<typeof ${theirKeyFn}> ? string : void)`;

        return `[${stringEscape(this.ourIndexName)}${ourTypeConstraint}, ${nonTermIndex}, ${stringEscape(this.theirIndexName)}${theirTypeConstraint}]`;
    }
}
NonTerminalConstraint.Equality = EqualityNonTerminalConstraint;

export class ConstantNonTerminalConstraint extends NonTerminalConstraint {
    constructor(public indexName : string,
                public valueCode : string) {
        super();
    }

    codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string) {
        if (ourKeyFn === 'undefined')
            console.error(`WARNING: key function is not set in constraint {${this.indexName} = ${this.valueCode}}, cannot check correctness statically`);
        const ourTypeConstraint = ourKeyFn === 'undefined' ? '' :
            ` as (${stringEscape(this.indexName)} extends keyof ReturnType<typeof ${ourKeyFn}> ? string : void)`;
        const constantTypeConstraint = ourKeyFn === 'undefined' ? '' :
            ` as ReturnType<typeof ${ourKeyFn}>[${stringEscape(this.indexName)}]`;

        return `[${stringEscape(this.indexName)}${ourTypeConstraint}, (${this.valueCode})${constantTypeConstraint}]`;
    }
}
NonTerminalConstraint.Constant = ConstantNonTerminalConstraint;

export abstract class RuleHeadPart {
    static NonTerminal : typeof NonTerminalRuleHead;
    static StringLiteral : typeof StringLiteralRuleHead;
    static ComputedStringLiteral : typeof ComputedStringLiteralRuleHead;
    static Choice : typeof ChoiceRuleHead;

    abstract visit(visitor : NodeVisitor) : void;
    abstract getTemplate() : string;
}

export class NonTerminalRuleHead extends RuleHeadPart {
    type = 'any';
    keyfn = 'undefined';

    constructor(public name : string|null,
                public symbol : string,
                public constraint : NonTerminalConstraint|null) {
        super();
    }

    toString() {
        return `${this.name} : NT[${this.symbol}]`;
    }

    visit(visitor : NodeVisitor) {
        visitor.visitNonTerminalRuleHead(this);
    }

    getTemplate() {
        return `\${${this.name ?? this.symbol}}`;
    }

    codegen(allNonTerminals : NonTerminalRuleHead[], index : number) : string {
        return `new $runtime.NonTerminal(${stringEscape(this.symbol)}, ${this.name !== null ? stringEscape(this.name) : 'undefined'}, ${this.constraint ? this.constraint.codegen(allNonTerminals, this.keyfn, index) : 'undefined'})`;
    }
}
RuleHeadPart.NonTerminal = NonTerminalRuleHead;

function templateEscape(str : string) {
    return str.replace(/[${}|[\]\\]/g, '\\$0');
}

export class StringLiteralRuleHead extends RuleHeadPart {
    constructor(public value : string) {
        super();
    }

    visit(visitor : NodeVisitor) {}

    getTemplate() {
        // note the double escaping here:
        // getTemplate() will escape any special character that have meaning to the template
        // language (so $, {, }, |, etc.)
        // but we also escape any special character that have meaning in JS, so the resulting
        // string can be output as a double-quoted JS string

        return templateEscape(this.value).replace(/(["\\])/g, '\\$1').replace(/\n/g, '\\n');
    }
}
RuleHeadPart.StringLiteral = StringLiteralRuleHead;

export class ComputedStringLiteralRuleHead extends RuleHeadPart {
    constructor(public code : string) {
        super();
    }

    getTemplate() {
        // hack: we need to close the template string, add some piece dynamically, and then reopen it
        return `" + (${this.code}) + "`;
    }

    visit(visitor : NodeVisitor) {}
}
RuleHeadPart.ComputedStringLiteral = ComputedStringLiteralRuleHead;

export class ChoiceRuleHead extends RuleHeadPart {
    constructor(public values : string[]) {
        super();
    }

    getTemplate() {
        return '{' + this.values.map((v) => templateEscape(v).replace(/(["\\])/g, '\\$1').replace(/\n/g, '\\n')).join('|') + '}';
    }

    visit(visitor : NodeVisitor) {}
}
RuleHeadPart.Choice = ChoiceRuleHead;

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as path from 'path';	97✔
22	import assert from 'assert';	97✔
23
24	import { stringEscape } from '../../utils/escaping';	97✔
25	import * as TemplateGrammar from '../../utils/template-string/grammar';	97✔
26	import * as I18n from '../../i18n';	97✔
27
28	export class NodeVisitor {	97✔
29	visitImport(stmt : Import) {}
30
31	visitContextStmt(stmt : ContextStmt) {}
32	visitNonTerminalStmt(stmt : NonTerminalStmt) {}
33	visitKeyFunctionDeclaration(stmt : KeyFunctionDeclarationStmt) {}
34
35	visitOldStyleExpansionRule(stmt : OldStyleExpansion) {}
36	visitNewStyleExpansionRule(stmt : NewStyleExpansion) {}
37	visitConstantsRule(stmt : Constants) {}
38	visitConditionRule(stmt : Condition) {}
39
40	visitNonTerminalRuleHead(node : RuleHeadPart) {}
41	}
42
43	export class Grammar {	97✔
44	constructor(public comment : string,	×
45	public statements : Statement[]) {	×
46	}
47
48	visit(visitor : NodeVisitor) {
49	for (const stmt of this.statements)	×
50	stmt.visit(visitor);	×
51	}
52
53	codegen(filename : string) : string {
54	let buffer = '';	×
55
56	buffer += (this.comment);	×
57	for (const stmt of this.statements) {	×
58	if (stmt instanceof JSImportStmt)	×
59	buffer += stmt.codegen();	×
60	}
61
NEW 62	let runtimepath = path.relative(path.dirname(filename), require.resolve('../runtime'));	×
NEW 63	if (runtimepath.endsWith('.ts'))	×
NEW 64	runtimepath = runtimepath.substring(0, runtimepath.length-3);	×
NEW 65	let generatorpath = path.relative(path.dirname(filename), require.resolve('../generator'));	×
NEW 66	if (generatorpath.endsWith('.ts'))	×
NEW 67	generatorpath = generatorpath.substring(0, generatorpath.length-3);	×
NEW 68	let typespath = path.relative(path.dirname(filename), require.resolve('../types'));	×
NEW 69	if (typespath.endsWith('.ts'))	×
NEW 70	typespath = typespath.substring(0, typespath.length-3);	×
NEW 71	let i18npath = path.relative(path.dirname(filename), require.resolve('../../i18n'));	×
NEW 72	if (i18npath.endsWith('.ts'))	×
NEW 73	i18npath = i18npath.substring(0, i18npath.length-3);	×
NEW 74	buffer += `import * as $runtime from ${stringEscape(runtimepath)};\n`;	×
NEW 75	buffer += `import type $SentenceGenerator from ${stringEscape(generatorpath)};\n`;	×
NEW 76	buffer += `import * as $SentenceGeneratorTypes from ${stringEscape(typespath)};\n`;	×
NEW 77	buffer += `import * as $I18n from ${stringEscape(i18npath)};\n`;	×
NEW 78	buffer += `export default async function($options : $SentenceGeneratorTypes.GrammarOptions, $locale : $I18n.LanguagePack, $grammar : $SentenceGenerator<any, any>, $loader : ThingpediaLoader) : Promise<void> {\n`;	×
79	for (const stmt of this.statements) {	×
80	if (stmt instanceof JSImportStmt)	×
81	continue;	×
82	buffer += stmt.codegen(' ');	×
83	}
84	buffer += '};\n';	×
85
86	return buffer;	×
87	}
88	}
89
90	export abstract class Statement {	97✔
91	static CodeBlock : typeof CodeBlock;
92	static JSImportStmt : typeof JSImportStmt;
93	static NonTerminal : typeof NonTerminalStmt;
94	static Context : typeof ContextStmt;
95	static FunctionDeclaration : typeof FunctionDeclarationStmt;
96	static ForLoop : typeof ForLoop;
97	static If : typeof IfStmt;
98	static Import : typeof Import;
99	static KeyFunctionDeclaration : typeof KeyFunctionDeclarationStmt;
100
101	abstract codegen(prefix ?: string) : string;
102	abstract visit(visitor : NodeVisitor) : void;
103	}
104
105	export class CodeBlock extends Statement {	97✔
106	constructor(public code : string) {	×
107	super();	×
108	}
109
110	codegen() : string {
111	return this.code;	×
112	}
113
114	visit(visitor : NodeVisitor) {}
115	}
116	Statement.CodeBlock = CodeBlock;	97✔
117
118	export class JSImportStmt extends Statement {	97✔
119	constructor(public code : string) {	×
120	super();	×
121	}
122
123	codegen() : string {
124	return `import ${this.code};\n`;	×
125	}
126
127	visit(visitor : NodeVisitor) {}
128	}
129	Statement.JSImportStmt = JSImportStmt;	97✔
130
131	export class KeyFunctionDeclarationStmt extends Statement {	97✔
132	constructor(public decls : Array<[string, string]>) {	×
133	super();	×
134	}
135
136	codegen() : string {
137	return ''; // this is a type declaration processed by the compiler,	×
138	// it generates no code
139	}
140
141	visit(visitor : NodeVisitor) {
142	visitor.visitKeyFunctionDeclaration(this);	×
143	}
144	}
145	Statement.KeyFunctionDeclaration = KeyFunctionDeclarationStmt;	97✔
146
147	export class NonTerminalStmt extends Statement {	97✔
148	keyfn = 'undefined';	×
149
150	constructor(public name : string,	×
151	public type : string\|undefined,	×
152	public rules : Rule[]) {	×
153	super();	×
154	}
155
156	visit(visitor : NodeVisitor) {
157	visitor.visitNonTerminalStmt(this);	×
158	for (const rule of this.rules)	×
159	rule.visit(visitor);	×
160	}
161
162	codegen(prefix = '') : string {	×
163	let buffer = '';	×
164	buffer += (`${prefix}$grammar.declareSymbol(${stringEscape(this.name)});\n`);	×
165	for (const rule of this.rules)	×
166	buffer += rule.codegen(this.name, prefix, this.type \|\| 'any', this.keyfn);	×
167	return buffer;	×
168	}
169	}
170	Statement.NonTerminal = NonTerminalStmt;	97✔
171
172	export class ContextStmt extends Statement {	97✔
173	constructor(public names : string[],	×
174	public type : string\|undefined) {	×
175	super();	×
176	}
177
178	visit(visitor : NodeVisitor) {
179	visitor.visitContextStmt(this);	×
180	}
181
182	codegen(prefix = '') : string {	×
183	return this.names.map((name) => `${prefix}$grammar.declareContext(${stringEscape(name)});\n`).join('');	×
184	}
185	}
186	Statement.Context = ContextStmt;	97✔
187
188	export class FunctionDeclarationStmt extends Statement {	97✔
189	constructor(public name : string,	×
190	public args : string,	×
191	public code : string,	×
192	public type = 'any') {	×
193	super();	×
194	}
195
196	visit(visitor : NodeVisitor) {}
197
198	codegen(prefix = '') : string {	×
199	return `${prefix}$grammar.declareFunction('${this.name}', (${this.args}) : ${this.type} => {${this.code}});\n`;	×
200	}
201	}
202	Statement.FunctionDeclaration = FunctionDeclarationStmt;	97✔
203
204	export class ForLoop extends Statement {	97✔
205	constructor(public head : string,	×
206	public statements : Statement[]) {	×
207	super();	×
208	}
209
210	visit(visitor : NodeVisitor) {
211	for (const stmt of this.statements)	×
212	stmt.visit(visitor);	×
213	}
214
215	codegen(prefix = '') : string {	×
216	let buffer = '';	×
217	buffer += (`${prefix}for (${this.head}) {\n`);	×
218	for (const stmt of this.statements)	×
219	buffer += stmt.codegen(prefix + ' ');	×
220	buffer += (`${prefix}}\n`);	×
221	return buffer;	×
222	}
223	}
224	Statement.ForLoop = ForLoop;	97✔
225
226	export class IfStmt extends Statement {	97✔
227	constructor(public cond : string,	×
228	public iftrue : Statement[],	×
229	public iffalse : Statement[]) {	×
230	super();	×
231	}
232
233	visit(visitor : NodeVisitor) {
234	for (const stmt of this.iftrue)	×
235	stmt.visit(visitor);	×
236	for (const stmt of this.iffalse)	×
237	stmt.visit(visitor);	×
238	}
239
240	codegen(prefix = '') : string {	×
241	let buffer = '';	×
242	buffer += (`${prefix}if (${this.cond}) {\n`);	×
243	for (const stmt of this.iftrue)	×
244	buffer += stmt.codegen(prefix + ' ');	×
245	if (this.iffalse.length > 0) {	×
246	buffer += (`${prefix}} else {\n`);	×
247	for (const stmt of this.iffalse)	×
248	buffer += stmt.codegen(prefix + ' ');	×
249	}
250	buffer += (`${prefix}}\n`);	×
251	return buffer;	×
252	}
253	}
254	Statement.If = IfStmt;	97✔
255
256	export class Import extends Statement {	97✔
257	constructor(public what : string) {	×
258	super();	×
259	}
260
261	visit(visitor : NodeVisitor) {
262	visitor.visitImport(this);	×
263	}
264
265	codegen(prefix = '') : string {	×
NEW 266	return `${prefix}await (await $runtime.import(${stringEscape(this.what)}, __dirname))($options, $locale, $grammar, $loader);\n`;	×
267	}
268	}
269	Statement.Import = Import;	97✔
270
271	interface RuleAttribute {
272	name : string;
273	code : string;
274	}
275
276	export class RuleAttributes {	97✔
277	constructor(public attributes : RuleAttribute[] = []) {	×
278	}
279
280	codegen() : string {
281	let buf = '{ ';	×
282	buf += this.attributes.map((attr) => `${attr.name}: (${attr.code})`).join(', ');	×
283	buf += ' }';	×
284	return buf;	×
285	}
286	}
287
288	export abstract class Rule {	97✔
289	static Constants : typeof Constants;
290	static Condition : typeof Condition;
291	static OldStyleExpansion : typeof OldStyleExpansion;
292	static NewStyleExpansion : typeof NewStyleExpansion;
293
294	abstract codegen(nonTerminal : string, prefix : string, type : string, keyfn : string) : string;
295	abstract visit(visitor : NodeVisitor) : void;
296	}
297
298	export class Constants extends Rule {	97✔
299	constructor(public token : string,	×
300	public typeCode : string,	×
301	public attrs : RuleAttributes) {	×
302	super();	×
303	}
304
305	visit(visitor : NodeVisitor) {
306	visitor.visitConstantsRule(this);	×
307	}
308
309	codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {	×
310	return `${prefix}$grammar.addConstants(${stringEscape(nonTerminal)}, ${stringEscape(this.token)}, ${this.typeCode}, ${keyfn}, ${this.attrs.codegen()});\n`;	×
311	}
312	}
313	Rule.Constants = Constants;	97✔
314
315	function makeBodyLambda(head : NonTerminalRuleHead[],
316	body : string,
317	type = 'any') : string {	×
318	const bodyArgs : string[] = [];	×
319	let i = 0;	×
320	for (const headPart of head) {	×
321	if (headPart instanceof NonTerminalRuleHead && headPart.name)	×
322	bodyArgs.push(headPart.name + ' : ' + headPart.type);	×
323	else
324	bodyArgs.push(`$${i++}` + ' : ' + headPart.type);	×
325	}
326
327	return `(${bodyArgs.join(', ')}) : (${type})\|null => ${body}`;	×
328	}
329
330	export class OldStyleExpansion extends Rule {	97✔
331	constructor(public head : RuleHeadPart[],	×
332	public bodyCode : string,	×
333	public attrs : RuleAttributes) {	×
334	super();	×
335	assert(Array.isArray(head));	×
336	}
337
338	visit(visitor : NodeVisitor) {
339	visitor.visitOldStyleExpansionRule(this);	×
340	for (const head of this.head)	×
341	head.visit(visitor);	×
342	}
343
344	codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {	×
345	const nonTerminalChildren : NonTerminalRuleHead[] = this.head.filter((h) : h is NonTerminalRuleHead => h instanceof NonTerminalRuleHead);	×
346	const expanderCode = makeBodyLambda(nonTerminalChildren, this.bodyCode, type);	×
347
348	let template = '"' + this.head.map((h) => h.getTemplate()).join(' ') + '"';	×
349
350	// generate code to lookup the translation of the template if meaningful
351	// (skip if this template has only one component)
352	if (this.head.length > 1 && template !== '""')	×
353	template = `$locale._(${template})`;	×
354
355	return `${prefix}$grammar.addRule(${stringEscape(nonTerminal)}, [${nonTerminalChildren.map((h, i) => h.codegen(nonTerminalChildren, i)).join(', ')}], ${template}, (${expanderCode}), ${keyfn}, ${this.attrs.codegen()});\n`;	×
356	}
357	}
358	Rule.OldStyleExpansion = OldStyleExpansion;	97✔
359
360	export class NewStyleExpansion extends Rule {	97✔
361	constructor(public nonTerminals : NonTerminalRuleHead[],	×
362	public sentenceTemplate : string,	×
363	public bodyCode : string,	×
364	public attrs : RuleAttributes) {	×
365	super();	×
366	}
367
368	visit(visitor : NodeVisitor) {
369	visitor.visitNewStyleExpansionRule(this);	×
370	for (const nt of this.nonTerminals)	×
371	nt.visit(visitor);	×
372	}
373
374	codegen(nonTerminal : string, prefix = '', type : string, keyfn : string) : string {	×
375	const expanderCode = makeBodyLambda(this.nonTerminals, this.bodyCode, type);	×
376
377	// try parsing the template and preprocessing, so we catch errors eagerly
378	try {	×
379	TemplateGrammar.parse(this.sentenceTemplate).preprocess(I18n.get('en-US'), this.nonTerminals.map((e) => e.name ?? e.symbol));	×
380	} catch(e) {
381	throw new Error(`Failed to parse template string for ${nonTerminal} = ${this.sentenceTemplate} (${this.nonTerminals.join(', ')}): ${e.message}`);	×
382	}
383
384	let gettext;
385	if (this.sentenceTemplate)	×
386	gettext = `$locale._(${stringEscape(this.sentenceTemplate)})`;	×
387	else
388	gettext = "''";	×
389
390	return `${prefix}$grammar.addRule(${stringEscape(nonTerminal)}, [${this.nonTerminals.map((h, i) => h.codegen(this.nonTerminals, i)).join(', ')}], ${gettext}, (${expanderCode}), ${keyfn}, ${this.attrs.codegen()});\n`;	×
391	}
392	}
393	Rule.NewStyleExpansion = NewStyleExpansion;	97✔
394
395	export class Condition extends Rule {	97✔
396	constructor(public flag : string,	×
397	public rules : Rule[]) {	×
398	super();	×
399	}
400
401	visit(visitor : NodeVisitor) {
402	visitor.visitConditionRule(this);	×
403	for (const rule of this.rules)	×
404	rule.visit(visitor);	×
405	}
406
407	codegen(nonTerminal : string, prefix : string, type : string, keyfn : string) : string {
408	const flag = this.flag.startsWith('?') ?	×
409	`$options.flags.${this.flag.substring(1)}` :	×
410	`!$options.flags.${this.flag.substring(1)}`;
411
412	let buffer = '';	×
413	buffer += (`${prefix}if (${flag}) {\n`);	×
414	for (const rule of this.rules)	×
415	buffer += rule.codegen(nonTerminal, prefix + ' ', type, keyfn);	×
416	buffer += (`${prefix}}\n`);	×
417	return buffer;	×
418	}
419	}
420	Rule.Condition = Condition;	97✔
421
422	export abstract class NonTerminalConstraint {	97✔
423	static Constant : typeof ConstantNonTerminalConstraint;
424	static Equality : typeof EqualityNonTerminalConstraint;
425
426	abstract codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string, ourIndex : number) : string;
427	}
428
429	export class EqualityNonTerminalConstraint extends NonTerminalConstraint {	97✔
430	constructor(public ourIndexName : string,	×
431	public nonTermRef : string,	×
432	public theirIndexName : string) {	×
433	super();	×
434	}
435
436	codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string, ourIndex : number) {
437	if (ourKeyFn === 'undefined')	×
438	console.error(`WARNING: key function is not set in constraint {${this.ourIndexName} = ${this.nonTermRef}.${this.theirIndexName}}, cannot check correctness statically`);	×
439	const ourTypeConstraint = ourKeyFn === 'undefined' ? '' :	×
440	` as (${stringEscape(this.ourIndexName)} extends keyof ReturnType<typeof ${ourKeyFn}> ? string : void)`;
441
442	let nonTermIndex, theirKeyFn;
443	if (/^[0-9]+/.test(this.nonTermRef)) {	×
444	nonTermIndex = parseInt(this.nonTermRef, 10);	×
445	assert(allNonTerminals[nonTermIndex]);	×
446
447	theirKeyFn = allNonTerminals[nonTermIndex];	×
448	} else {
449	for (let i = 0; i < allNonTerminals.length; i++) {	×
450	const part = allNonTerminals[i];	×
451	if (part.name === this.nonTermRef) {	×
452	nonTermIndex = i;	×
453	theirKeyFn = part.keyfn;	×
454	break;	×
455	}
456	}
457	if (nonTermIndex === undefined)	×
458	throw new Error(`Invalid non-terminal backreference to ${this.nonTermRef} for equality constraint of ${allNonTerminals[ourIndex]} (alias not found)`);	×
459	if (nonTermIndex >= ourIndex)	×
460	throw new Error(`Invalid non-terminal backreference to ${this.nonTermRef} for equality constraint of ${allNonTerminals[ourIndex]} (alias must precede usage)`);	×
461	}
462	if (theirKeyFn === 'undefined')	×
463	console.error(`WARNING: key function is not set in constraint {${this.ourIndexName} = ${this.nonTermRef}.${this.theirIndexName}}, cannot check correctness statically`);	×
464	const theirTypeConstraint = theirKeyFn === 'undefined' ? '' :	×
465	` as (${stringEscape(this.theirIndexName)} extends keyof ReturnType<typeof ${theirKeyFn}> ? string : void)`;
466
467	return `[${stringEscape(this.ourIndexName)}${ourTypeConstraint}, ${nonTermIndex}, ${stringEscape(this.theirIndexName)}${theirTypeConstraint}]`;	×
468	}
469	}
470	NonTerminalConstraint.Equality = EqualityNonTerminalConstraint;	97✔
471
472	export class ConstantNonTerminalConstraint extends NonTerminalConstraint {	97✔
473	constructor(public indexName : string,	×
474	public valueCode : string) {	×
475	super();	×
476	}
477
478	codegen(allNonTerminals : NonTerminalRuleHead[], ourKeyFn : string) {
479	if (ourKeyFn === 'undefined')	×
480	console.error(`WARNING: key function is not set in constraint {${this.indexName} = ${this.valueCode}}, cannot check correctness statically`);	×
481	const ourTypeConstraint = ourKeyFn === 'undefined' ? '' :	×
482	` as (${stringEscape(this.indexName)} extends keyof ReturnType<typeof ${ourKeyFn}> ? string : void)`;
483	const constantTypeConstraint = ourKeyFn === 'undefined' ? '' :	×
484	` as ReturnType<typeof ${ourKeyFn}>[${stringEscape(this.indexName)}]`;
485
486	return `[${stringEscape(this.indexName)}${ourTypeConstraint}, (${this.valueCode})${constantTypeConstraint}]`;	×
487	}
488	}
489	NonTerminalConstraint.Constant = ConstantNonTerminalConstraint;	97✔
490
491	export abstract class RuleHeadPart {	97✔
492	static NonTerminal : typeof NonTerminalRuleHead;
493	static StringLiteral : typeof StringLiteralRuleHead;
494	static ComputedStringLiteral : typeof ComputedStringLiteralRuleHead;
495	static Choice : typeof ChoiceRuleHead;
496
497	abstract visit(visitor : NodeVisitor) : void;
498	abstract getTemplate() : string;
499	}
500
501	export class NonTerminalRuleHead extends RuleHeadPart {	97✔
502	type = 'any';	×
503	keyfn = 'undefined';	×
504
505	constructor(public name : string\|null,	×
506	public symbol : string,	×
507	public constraint : NonTerminalConstraint\|null) {	×
508	super();	×
509	}
510
511	toString() {
512	return `${this.name} : NT[${this.symbol}]`;	×
513	}
514
515	visit(visitor : NodeVisitor) {
516	visitor.visitNonTerminalRuleHead(this);	×
517	}
518
519	getTemplate() {
520	return `\${${this.name ?? this.symbol}}`;	×
521	}
522
523	codegen(allNonTerminals : NonTerminalRuleHead[], index : number) : string {
524	return `new $runtime.NonTerminal(${stringEscape(this.symbol)}, ${this.name !== null ? stringEscape(this.name) : 'undefined'}, ${this.constraint ? this.constraint.codegen(allNonTerminals, this.keyfn, index) : 'undefined'})`;	×
525	}
526	}
527	RuleHeadPart.NonTerminal = NonTerminalRuleHead;	97✔
528
529	function templateEscape(str : string) {
530	return str.replace(/[${}\|[\]\\]/g, '\\$0');	×
531	}
532
533	export class StringLiteralRuleHead extends RuleHeadPart {	97✔
534	constructor(public value : string) {	×
535	super();	×
536	}
537
538	visit(visitor : NodeVisitor) {}
539
540	getTemplate() {
541	// note the double escaping here:
542	// getTemplate() will escape any special character that have meaning to the template
543	// language (so $, {, }, \|, etc.)
544	// but we also escape any special character that have meaning in JS, so the resulting
545	// string can be output as a double-quoted JS string
546
547	return templateEscape(this.value).replace(/(["\\])/g, '\\$1').replace(/\n/g, '\\n');	×
548	}
549	}
550	RuleHeadPart.StringLiteral = StringLiteralRuleHead;	97✔
551
552	export class ComputedStringLiteralRuleHead extends RuleHeadPart {	97✔
553	constructor(public code : string) {	×
554	super();	×
555	}
556
557	getTemplate() {
558	// hack: we need to close the template string, add some piece dynamically, and then reopen it
559	return `" + (${this.code}) + "`;	×
560	}
561
562	visit(visitor : NodeVisitor) {}
563	}
564	RuleHeadPart.ComputedStringLiteral = ComputedStringLiteralRuleHead;	97✔
565
566	export class ChoiceRuleHead extends RuleHeadPart {	97✔
567	constructor(public values : string[]) {	×
568	super();	×
569	}
570
571	getTemplate() {
572	return '{' + this.values.map((v) => templateEscape(v).replace(/(["\\])/g, '\\$1').replace(/\n/g, '\\n')).join('\|') + '}';	×
573	}
574
575	visit(visitor : NodeVisitor) {}
576	}
577	RuleHeadPart.Choice = ChoiceRuleHead;	97✔

stanford-oval / genie-toolkit / 2055

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous