2043

pending completion

Build # 2043

Build Type

Pull #636

travis-ci-com

Committed by

web-flow

Commit Message

Merge 8c46af48b into 75988e26e

Pull Request Pull Request #636: Clean up the template build system

Run Details

9846 of 19864 branches covered (49.57%)

Branch coverage included in aggregate %.

45 of 75 new or added lines in 9 files covered. (60.0%)

1 existing line in 1 file now uncovered.

23271 of 39210 relevant lines covered (59.35%)

523348.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.97

/lib/sentence-generator/compiler/index.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import { promises as pfs } from 'fs';
import * as path from 'path';
import * as ts from 'typescript';

import * as metagrammar from './grammar';
import * as metaast from './meta_ast';

import type { GrammarOptions } from '../types';
import type * as I18n from '../../i18n';
import type SentenceGenerator from '../generator';

const COMPILER_OPTIONS : ts.CompilerOptions = {
    module: ts.ModuleKind.CommonJS,
    target: ts.ScriptTarget.ES2018,
    allowJs: true,
    esModuleInterop: true,
    moduleResolution: ts.ModuleResolutionKind.NodeJs,
    resolveJsonModule: true,
    noUnusedLocals: false,
    strict: false,
};

export class Compiler {
    private _target : 'js' | 'ts';

    private _files = new Map<string, metaast.Grammar>();

    private _nonTerm = new Set<string>();
    // map a non-terminal to its type declaration, if any
    private _typeMap = new Map<string, string>();

    // map a type to its key function, if any
    private _keyFnMap = new Map<string, string>();

    constructor(target : 'js' | 'ts') {
        this._target = target;
    }

    private async _loadFile(filename : string) {
        if (this._files.has(filename))
            return;

        const dirname = path.dirname(filename);
        const input = await pfs.readFile(filename, { encoding: 'utf8' });
        let parsed;
        try {
            parsed = metagrammar.parse(input) as metaast.Grammar;
        } catch(e) {
            e.fileName = filename;
            throw e;
        }

        this._files.set(filename, parsed);

        const allImports = new Set<string>();
        const self = this;
        parsed.visit(new class extends metaast.NodeVisitor {
            visitImport(stmt : metaast.Import) {
                const resolved = path.resolve(dirname, stmt.what);
                allImports.add(resolved);
            }

            visitContextStmt(stmt : metaast.ContextStmt) {
                for (const symbol of stmt.names)
                    self._nonTerm.add(symbol);
                if (!stmt.type)
                    return;
                for (const symbol of stmt.names)
                    self._typeMap.set(symbol, stmt.type);
            }

            visitNonTerminalStmt(stmt : metaast.NonTerminalStmt) {
                const symbol = stmt.name;
                self._nonTerm.add(symbol);

                if (stmt.type === undefined || stmt.type === 'any')
                    return;

                const existing = self._typeMap.get(symbol);
                if (!existing || existing === 'any') {
                    self._typeMap.set(symbol, stmt.type);
                    return;
                }
                if (existing !== stmt.type)
                    throw new TypeError(`Invalid conflicting type annotation for non-terminal ${symbol}, have ${existing} want ${stmt.type}`);
            }

            visitKeyFunctionDeclaration(stmt : metaast.KeyFunctionDeclarationStmt) {
                for (let [type, keyfn] of stmt.decls) {
                    type = type.trim();
                    keyfn = keyfn.trim();

                    const existing = self._keyFnMap.get(type);
                    if (!existing) {
                        self._keyFnMap.set(type, keyfn);
                        continue;
                    }
                    if (existing !== keyfn)
                        throw new TypeError(`Invalid conflicting key function declaration for type ${type}`);
                }
            }
        });

        for (const import_ of allImports)
            await this._loadFile(import_);
    }

    private _assignAllTypes() {
        const self = this;
        this.visit(new class extends metaast.NodeVisitor {
            // assign a type to every usage of a non-terminal
            visitNonTerminalRuleHead(node : metaast.NonTerminalRuleHead) {
                const symbol = node.symbol;
                if (!self._nonTerm.has(symbol))
                    throw new TypeError(`Undeclared non-terminal ${symbol}`);

                const type = self._typeMap.get(symbol);
                if (type) {
                    node.type = type;
                    node.keyfn = self._keyFnMap.get(type.trim()) || 'undefined';
                }
                if (node.constraint && node.keyfn === 'undefined')
                    console.log(`WARNING: missing key function for type ${node.type}, which is used in constraint for non-terminal ${symbol}`);
            }

            // also assign a type to every non-terminal declaration, if
            // it doesn't have one already
            visitNonTerminalStmt(stmt : metaast.NonTerminalStmt) {
                if (stmt.type) {
                    stmt.keyfn = self._keyFnMap.get(stmt.type.trim()) || 'undefined';
                    return;
                }
                const symbol = stmt.name;

                const existing = self._typeMap.get(symbol);
                if (existing) {
                    stmt.type = existing;
                    stmt.keyfn = self._keyFnMap.get(existing.trim()) || 'undefined';
                }
            }
        });
    }

    visit(visitor : metaast.NodeVisitor) {
        for (const parsed of this._files.values())
            parsed.visit(visitor);
    }

    async parse(filename : string) {
        // load all template files and extract all the type annotations
        await this._loadFile(filename);

        // assign the type annotations to all the uses of the non-terminals
        this._assignAllTypes();
    }

    async process(filename : string) : Promise<void> {
        await this.parse(filename);
        await this._outputAllFiles();
    }

    private async _outputAllFiles() {
        for (const [filename, parsed] of this._files)
            await this._outputFile(filename, parsed);
    }

    private async _outputFile(filename : string,
                              parsed : metaast.Grammar) {
        const outputFile = filename + '.' + this._target;
        let output = parsed.codegen(filename);

        if (this._target === 'js') {
            const result = ts.transpileModule(output, {
                compilerOptions: COMPILER_OPTIONS,
                fileName: filename,
                reportDiagnostics: true,
            });
            if (result.diagnostics && result.diagnostics.length > 0) {
                let error = '';
                result.diagnostics.forEach((diagnostic) => {
                    if (diagnostic.file) {
                        const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start!);
                        const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');
                        error += `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}\n`;
                    } else {
                        error += ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n') + '\n';
                    }
                });
                throw new Error(`TypeScript compilation failed: ${error}`);
            }
            output = result.outputText;
        }

        await pfs.writeFile(outputFile, output);
    }
}


export function compile(filename : string) : Promise<void> {
    return new Compiler('ts').process(filename);
}

type CompiledTemplate = (options : GrammarOptions,
                         langPack : I18n.LanguagePack,
                         grammar : SentenceGenerator<any, any, any>,
                         loader ?: any) => Promise<void>;

export async function importGenie(filename : string,
                                  searchPath = path.resolve(path.dirname(module.filename), '../../templates')) : Promise<CompiledTemplate> {
    filename = path.resolve(searchPath, filename);

    // try loading compiled js first
    let target : 'js'|'ts' = 'js';
    try {
        if (filename.endsWith('.js'))
            return (await import(filename)).default;
        else
            return (await import(filename + '.' + target)).default;
    } catch(e) {
        if (e.code !== 'MODULE_NOT_FOUND')
            throw e;
    }

    // if that did not work, try compiling the template on-demand
    // we compile to .ts if we're running in ts-node or nyc with the
    // typescript extensions
    // and compile to js ourselves otherwise
    // (in the latter case, type errors won't be reported across modules)

    target = require.extensions['.ts'] ? 'ts' : 'js';
    try {
        await pfs.access(filename + '.' + target);
    } catch(e) {
        if (e.code !== 'ENOENT')
            throw e;
        await new Compiler(target).process(filename);
    }

    return (await import(filename + '.' + target)).default;
}

async function main() {
    try {
        await compile(process.argv[2]);
    } catch(e) {
        if (e.name === 'SyntaxError') {
            console.error(`Syntax error in ${e.fileName} at line ${e.location.start.line}: ${e.message}`);
        } else {
            console.error(e.message);
            console.error(e.stack);
        }
        process.exit(1);
    }
}
if (!module.parent)
    main();

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import { promises as pfs } from 'fs';	97✔
22	import * as path from 'path';	97✔
23	import * as ts from 'typescript';	97✔
24
25	import * as metagrammar from './grammar';	97✔
26	import * as metaast from './meta_ast';	97✔
27
28	import type { GrammarOptions } from '../types';
29	import type * as I18n from '../../i18n';
30	import type SentenceGenerator from '../generator';
31
32	const COMPILER_OPTIONS : ts.CompilerOptions = {	97✔
33	module: ts.ModuleKind.CommonJS,
34	target: ts.ScriptTarget.ES2018,
35	allowJs: true,
36	esModuleInterop: true,
37	moduleResolution: ts.ModuleResolutionKind.NodeJs,
38	resolveJsonModule: true,
39	noUnusedLocals: false,
40	strict: false,
41	};
42
43	export class Compiler {	97✔
44	private _target : 'js' \| 'ts';
45
46	private _files = new Map<string, metaast.Grammar>();	×
47
48	private _nonTerm = new Set<string>();	×
49	// map a non-terminal to its type declaration, if any
50	private _typeMap = new Map<string, string>();	×
51
52	// map a type to its key function, if any
53	private _keyFnMap = new Map<string, string>();	×
54
55	constructor(target : 'js' \| 'ts') {
56	this._target = target;	×
57	}
58
59	private async _loadFile(filename : string) {
60	if (this._files.has(filename))	×
61	return;	×
62
63	const dirname = path.dirname(filename);	×
64	const input = await pfs.readFile(filename, { encoding: 'utf8' });	×
65	let parsed;
66	try {	×
67	parsed = metagrammar.parse(input) as metaast.Grammar;	×
68	} catch(e) {
69	e.fileName = filename;	×
70	throw e;	×
71	}
72
73	this._files.set(filename, parsed);	×
74
75	const allImports = new Set<string>();	×
76	const self = this;	×
77	parsed.visit(new class extends metaast.NodeVisitor {	×
78	visitImport(stmt : metaast.Import) {
79	const resolved = path.resolve(dirname, stmt.what);	×
80	allImports.add(resolved);	×
81	}
82
83	visitContextStmt(stmt : metaast.ContextStmt) {
84	for (const symbol of stmt.names)	×
85	self._nonTerm.add(symbol);	×
86	if (!stmt.type)	×
87	return;	×
88	for (const symbol of stmt.names)	×
89	self._typeMap.set(symbol, stmt.type);	×
90	}
91
92	visitNonTerminalStmt(stmt : metaast.NonTerminalStmt) {
93	const symbol = stmt.name;	×
94	self._nonTerm.add(symbol);	×
95
96	if (stmt.type === undefined \|\| stmt.type === 'any')	×
97	return;	×
98
99	const existing = self._typeMap.get(symbol);	×
100	if (!existing \|\| existing === 'any') {	×
101	self._typeMap.set(symbol, stmt.type);	×
102	return;	×
103	}
104	if (existing !== stmt.type)	×
105	throw new TypeError(`Invalid conflicting type annotation for non-terminal ${symbol}, have ${existing} want ${stmt.type}`);	×
106	}
107
108	visitKeyFunctionDeclaration(stmt : metaast.KeyFunctionDeclarationStmt) {
109	for (let [type, keyfn] of stmt.decls) {	×
110	type = type.trim();	×
111	keyfn = keyfn.trim();	×
112
113	const existing = self._keyFnMap.get(type);	×
114	if (!existing) {	×
115	self._keyFnMap.set(type, keyfn);	×
116	continue;	×
117	}
118	if (existing !== keyfn)	×
119	throw new TypeError(`Invalid conflicting key function declaration for type ${type}`);	×
120	}
121	}
122	});
123
124	for (const import_ of allImports)	×
125	await this._loadFile(import_);	×
126	}
127
128	private _assignAllTypes() {
129	const self = this;	×
130	this.visit(new class extends metaast.NodeVisitor {	×
131	// assign a type to every usage of a non-terminal
132	visitNonTerminalRuleHead(node : metaast.NonTerminalRuleHead) {
133	const symbol = node.symbol;	×
134	if (!self._nonTerm.has(symbol))	×
135	throw new TypeError(`Undeclared non-terminal ${symbol}`);	×
136
137	const type = self._typeMap.get(symbol);	×
138	if (type) {	×
139	node.type = type;	×
140	node.keyfn = self._keyFnMap.get(type.trim()) \|\| 'undefined';	×
141	}
142	if (node.constraint && node.keyfn === 'undefined')	×
143	console.log(`WARNING: missing key function for type ${node.type}, which is used in constraint for non-terminal ${symbol}`);	×
144	}
145
146	// also assign a type to every non-terminal declaration, if
147	// it doesn't have one already
148	visitNonTerminalStmt(stmt : metaast.NonTerminalStmt) {
149	if (stmt.type) {	×
150	stmt.keyfn = self._keyFnMap.get(stmt.type.trim()) \|\| 'undefined';	×
151	return;	×
152	}
153	const symbol = stmt.name;	×
154
155	const existing = self._typeMap.get(symbol);	×
156	if (existing) {	×
157	stmt.type = existing;	×
158	stmt.keyfn = self._keyFnMap.get(existing.trim()) \|\| 'undefined';	×
159	}
160	}
161	});
162	}
163
164	visit(visitor : metaast.NodeVisitor) {
165	for (const parsed of this._files.values())	×
166	parsed.visit(visitor);	×
167	}
168
169	async parse(filename : string) {
170	// load all template files and extract all the type annotations
171	await this._loadFile(filename);	×
172
173	// assign the type annotations to all the uses of the non-terminals
174	this._assignAllTypes();	×
175	}
176
177	async process(filename : string) : Promise<void> {
178	await this.parse(filename);	×
179	await this._outputAllFiles();	×
180	}
181
182	private async _outputAllFiles() {
183	for (const [filename, parsed] of this._files)	×
184	await this._outputFile(filename, parsed);	×
185	}
186
187	private async _outputFile(filename : string,
188	parsed : metaast.Grammar) {
189	const outputFile = filename + '.' + this._target;	×
NEW 190	let output = parsed.codegen(filename);	×
191
192	if (this._target === 'js') {	×
193	const result = ts.transpileModule(output, {	×
194	compilerOptions: COMPILER_OPTIONS,
195	fileName: filename,
196	reportDiagnostics: true,
197	});
198	if (result.diagnostics && result.diagnostics.length > 0) {	×
199	let error = '';	×
200	result.diagnostics.forEach((diagnostic) => {	×
201	if (diagnostic.file) {	×
202	const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start!);	×
203	const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n');	×
204	error += `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}\n`;	×
205	} else {
206	error += ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n') + '\n';	×
207	}
208	});
209	throw new Error(`TypeScript compilation failed: ${error}`);	×
210	}
211	output = result.outputText;	×
212	}
213
214	await pfs.writeFile(outputFile, output);	×
215	}
216	}
217
218
219	export function compile(filename : string) : Promise<void> {	97✔
220	return new Compiler('ts').process(filename);	×
221	}
222
223	type CompiledTemplate = (options : GrammarOptions,
224	langPack : I18n.LanguagePack,
225	grammar : SentenceGenerator<any, any, any>,
226	loader ?: any) => Promise<void>;
227
228	export async function importGenie(filename : string,	97✔
229	searchPath = path.resolve(path.dirname(module.filename), '../../templates')) : Promise<CompiledTemplate> {	58✔
230	filename = path.resolve(searchPath, filename);	1,536✔
231
232	// try loading compiled js first
233	let target : 'js'\|'ts' = 'js';	1,536✔
234	try {	1,536✔
235	if (filename.endsWith('.js'))	1,536!
UNCOV 236	return (await import(filename)).default;	×
237	else
238	return (await import(filename + '.' + target)).default;	1,536✔
239	} catch(e) {
240	if (e.code !== 'MODULE_NOT_FOUND')	1,306!
241	throw e;	×
242	}
243
244	// if that did not work, try compiling the template on-demand
245	// we compile to .ts if we're running in ts-node or nyc with the
246	// typescript extensions
247	// and compile to js ourselves otherwise
248	// (in the latter case, type errors won't be reported across modules)
249
250	target = require.extensions['.ts'] ? 'ts' : 'js';	1,306!
251	try {	1,306✔
252	await pfs.access(filename + '.' + target);	1,306✔
253	} catch(e) {
254	if (e.code !== 'ENOENT')	×
255	throw e;	×
256	await new Compiler(target).process(filename);	×
257	}
258
259	return (await import(filename + '.' + target)).default;	1,306✔
260	}
261
262	async function main() {
NEW 263	try {	×
NEW 264	await compile(process.argv[2]);	×
265	} catch(e) {
NEW 266	if (e.name === 'SyntaxError') {	×
NEW 267	console.error(`Syntax error in ${e.fileName} at line ${e.location.start.line}: ${e.message}`);	×
268	} else {
NEW 269	console.error(e.message);	×
NEW 270	console.error(e.stack);	×
271	}
NEW 272	process.exit(1);	×
273	}
274	}
275	if (!module.parent)	97!
NEW 276	main();	×

stanford-oval / genie-toolkit / 2043

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous