stanford-oval / thingtalk / 986

pending completion

Build # 986

Build Type

Pull #232

travis-ci-com

Committed by

web-flow

Commit Message

Merge 82a31ac24 into 5e815cadb

Pull Request Pull Request #232: Add DatePiece class

Run Details

4370 of 5837 branches covered (74.87%)

Branch coverage included in aggregate %.

68 of 77 new or added lines in 5 files covered. (88.31%)

17 existing lines in 2 files now uncovered.

8579 of 10277 relevant lines covered (83.48%)

1556.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.53

/lib/compiler/index.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of ThingTalk
//
// Copyright 2018-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const Grammar = require('../grammar');
const { typeCheckProgram } = require('../typecheck');
const { NotCompilableError, NotImplementedError } = require('../errors');
const Ast = require('../ast');

const JSIr = require('./jsir');
const { compileStatementToOp, compileStreamToOps, compileTableToOps } = require('./ast-to-ops');
const { QueryInvocationHints } = require('./ops');
const { getDefaultProjection } = require('./utils');
const OpCompiler = require('./ops-to-jsir');
const Scope = require('./scope');

class CompiledProgram {
    constructor(states, command, rules) {
        this.hasTrigger = rules.length > 0;

        this.states = states;
        this.command = command;
        this.rules = rules;
    }
}

module.exports = class AppCompiler {
    constructor(schemaRetriever, testMode) {
        this._testMode = testMode;
        this._declarations = null;
        this._toplevelscope = {};

        this._schemaRetriever = schemaRetriever;
        this._nextStateVar = 0;
        this._nextProcId = 0;

        this._astVars = [];
    }

    compileCode(code) {
        return this.compileProgram(Grammar.parse(code));
    }

    _allocState() {
        return this._nextStateVar++;
    }

    _allocAst(v) {
        this._astVars.push(v);
        return this._astVars.length-1;
    }

    _getDeclaration(name) {
        return this._declarations.get(name);
    }

    _getCurrentScope() {
        let flatscope = {};
        for (let [name, value] of this._declarations) {
            if (value.type === 'declaration')
                flatscope[name] = value.code;
        }
        return flatscope;
    }

    _declareArguments(args, scope, irBuilder) {
        const compiledArgs = [];

        for (let name in args) {
            const reg = irBuilder.allocArgument();
            scope.set(name, {
                type: 'scalar',
                tt_type: args[name],
                register: reg
            });
            compiledArgs.push(name);
        }

        return compiledArgs;
    }

    _compileDeclarationFunction(decl, parentIRBuilder) {
        const irBuilder = new JSIr.IRBuilder(parentIRBuilder ? parentIRBuilder.nextRegister : 0, ['__emit']);

        const functionScope = new Scope(this._declarations);
        const args = this._declareArguments(decl.args, functionScope, irBuilder);
        const opCompiler = new OpCompiler(this, functionScope, irBuilder);
        let op;

        const hints = new QueryInvocationHints(new Set(getDefaultProjection(decl.schema)));

        switch (decl.type) {
        case 'query':
            op = compileTableToOps(decl.value, [], hints);
            opCompiler.compileQueryDeclaration(op);
            break;

        case 'stream':
            op = compileStreamToOps(decl.value, hints);
            opCompiler.compileStreamDeclaration(op);
            break;

        case 'action':
            opCompiler.compileActionDeclaration(decl.value);
            break;
        }

        let code;
        let register;
        if (parentIRBuilder) {
            parentIRBuilder.skipRegisterRange(irBuilder.registerRange);

            register = parentIRBuilder.allocRegister();
            parentIRBuilder.add(new JSIr.AsyncFunctionDeclaration(register, irBuilder));
            code = null;
        } else {
            register = null;
            code = this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);
            this._toplevelscope[decl.name] = code;
        }

        this._declarations.set(decl.name, {
            type: 'declaration',
            args, code, register,
            schema: decl.schema,
        });
    }

    _compileAssignment(assignment, irBuilder, { hasAnyStream, forProcedure}) {
        const opCompiler = new OpCompiler(this, this._declarations, irBuilder, forProcedure);

        // at the top level, assignments can be referred to by streams, so
        // they need to be persistent (save to disk) such that when the program
        // is restarted, the result can be reused.
        // (this is only needed in top-level since stream is not allowed within
        // procedures)
        const isPersistent = hasAnyStream;

        let register;

        if (assignment.isAction) {
            register = opCompiler.compileActionAssignment(assignment.value, isPersistent);
        } else {
            const hints = new QueryInvocationHints(new Set(getDefaultProjection(assignment.value.schema)));
            const tableop = compileTableToOps(assignment.value, [], hints);
            register = opCompiler.compileAssignment(tableop, isPersistent);
        }

        this._declarations.set(assignment.name, {
            type: 'assignment',
            isPersistent, register,
            schema: assignment.schema
        });
    }

    _compileProcedure(decl, parentIRBuilder) {
        const saveScope = this._declarations;

        const irBuilder = new JSIr.IRBuilder(parentIRBuilder ? parentIRBuilder.nextRegister : 0, ['__emit']);

        const procid = this._nextProcId++;
        irBuilder.setBeginEndHooks(
            new JSIr.EnterProcedure(procid, decl.name),
            new JSIr.ExitProcedure(procid, decl.name)
        );

        const procedureScope = new Scope(this._declarations);
        const args = this._declareArguments(decl.args, procedureScope, irBuilder);
        this._declarations = procedureScope;

        this._compileInScope(decl.value, decl.value.rules, irBuilder, {
            hasAnyStream: false,
            forProcedure: true
        });

        let code;
        let register;
        if (parentIRBuilder) {
            parentIRBuilder.skipRegisterRange(irBuilder.registerRange);

            register = parentIRBuilder.allocRegister();
            parentIRBuilder.add(new JSIr.AsyncFunctionDeclaration(register, irBuilder));
            code = null;
        } else {
            register = null;
            code = this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);
            this._toplevelscope[decl.name] = code;
        }

        this._declarations = saveScope;

        this._declarations.set(decl.name, {
            type: 'procedure',
            args, code, register,
            schema: decl.schema,
        });
    }

    _doCompileStatement(stmt, irBuilder, forProcedure) {
        const opCompiler = new OpCompiler(this, this._declarations, irBuilder, forProcedure);
        let ruleop = compileStatementToOp(stmt);
        opCompiler.compileStatement(ruleop);
    }

    _compileRule(rule) {
        // each rule goes into its own JS function
        const irBuilder = new JSIr.IRBuilder();
        this._doCompileStatement(rule, irBuilder);

        return this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);
    }

    _compileInScope(program, stmts, irBuilder, { hasAnyStream, forProcedure }) {
        for (let decl of program.declarations) {
            if (['stream', 'query', 'action'].indexOf(decl.type) >= 0)
                this._declarations[decl.name] = this._compileDeclarationFunction(decl, irBuilder);
            else if (decl.type === 'procedure')
                this._compileProcedure(decl, irBuilder);
            else
                throw new NotImplementedError(decl);
        }

        if (stmts.length === 0)
            return null;

        // all immediate statements are compiled into a single function, so we
        // create a single irBuilder that we share
        if (!irBuilder) {
            irBuilder = new JSIr.IRBuilder();
            const procid = this._nextProcId++;
            irBuilder.setBeginEndHooks(
                new JSIr.EnterProcedure(procid),
                new JSIr.ExitProcedure(procid)
            );
        }
        
        for (let i = 0; i < stmts.length; i++) {
            // if this is not the first statement, clear the get cache before running it
            if (i !== 0)
                irBuilder.add(new JSIr.ClearGetCache());
            if (stmts[i].isAssignment)
                this._compileAssignment(stmts[i], irBuilder, { hasAnyStream, forProcedure });
            else
                this._doCompileStatement(stmts[i], irBuilder, forProcedure);
        }

        return irBuilder;
    }

    _verifyCompilable(program) {
        if (program.principal !== null)
            throw new NotCompilableError(`Remote programs cannot be compiled, they must be sent to a different runtime instead`);

        for (let slot of program.iterateSlots2()) {
            if (slot instanceof Ast.Selector) {
                if (slot.principal !== null)
                    throw new NotCompilableError(`Remote primitives cannot be compiled, they must be lowered and sent to a different runtime instead`);
                continue;
            }
            if (!slot.isCompilable())
                throw new NotCompilableError(`Programs with slots or unresolved values cannot be compiled, and must be slot-filled first`);
        }
    }

    async compileProgram(program) {
        await typeCheckProgram(program, this._schemaRetriever);
        this._verifyCompilable(program);

        const compiledRules = [];
        const immediate = [];
        const rules = [];

        for (let stmt of program.rules) {
            if (stmt.isAssignment || stmt.isCommand)
                immediate.push(stmt);
            else
                rules.push(stmt);
        }

        this._declarations = new Scope;
        const commandIRBuilder = this._compileInScope(program, immediate, null, {
            hasAnyStream: rules.length > 0,
            forProcedure: false
        });
        let compiledCommand;
        if (commandIRBuilder !== null) {
            if (this._testMode)
                compiledCommand = commandIRBuilder.codegen();
            else
                compiledCommand = commandIRBuilder.compile(this._toplevelscope, this._astVars);
        } else {
            compiledCommand = null;
        }
        for (let rule of rules)
            compiledRules.push(this._compileRule(rule));

        return new CompiledProgram(this._nextStateVar, compiledCommand, compiledRules);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of ThingTalk
4	//
5	// Copyright 2018-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const Grammar = require('../grammar');	2✔
23	const { typeCheckProgram } = require('../typecheck');	2✔
24	const { NotCompilableError, NotImplementedError } = require('../errors');	2✔
25	const Ast = require('../ast');	2✔
26
27	const JSIr = require('./jsir');	2✔
28	const { compileStatementToOp, compileStreamToOps, compileTableToOps } = require('./ast-to-ops');	2✔
29	const { QueryInvocationHints } = require('./ops');	2✔
30	const { getDefaultProjection } = require('./utils');	2✔
31	const OpCompiler = require('./ops-to-jsir');	2✔
32	const Scope = require('./scope');	2✔
33
34	class CompiledProgram {
35	constructor(states, command, rules) {
36	this.hasTrigger = rules.length > 0;	320✔
37
38	this.states = states;	320✔
39	this.command = command;	320✔
40	this.rules = rules;	320✔
41	}
42	}
43
44	module.exports = class AppCompiler {	2✔
45	constructor(schemaRetriever, testMode) {
46	this._testMode = testMode;	320✔
47	this._declarations = null;	320✔
48	this._toplevelscope = {};	320✔
49
50	this._schemaRetriever = schemaRetriever;	320✔
51	this._nextStateVar = 0;	320✔
52	this._nextProcId = 0;	320✔
53
54	this._astVars = [];	320✔
55	}
56
57	compileCode(code) {
58	return this.compileProgram(Grammar.parse(code));	304✔
59	}
60
61	_allocState() {
62	return this._nextStateVar++;	94✔
63	}
64
65	_allocAst(v) {
66	this._astVars.push(v);	10✔
67	return this._astVars.length-1;	10✔
68	}
69
70	_getDeclaration(name) {
71	return this._declarations.get(name);	×
72	}
73
74	_getCurrentScope() {
75	let flatscope = {};	×
76	for (let [name, value] of this._declarations) {	×
77	if (value.type === 'declaration')	×
78	flatscope[name] = value.code;	×
79	}
80	return flatscope;	×
81	}
82
83	_declareArguments(args, scope, irBuilder) {
84	const compiledArgs = [];	46✔
85
86	for (let name in args) {	46✔
87	const reg = irBuilder.allocArgument();	40✔
88	scope.set(name, {	40✔
89	type: 'scalar',
90	tt_type: args[name],
91	register: reg
92	});
93	compiledArgs.push(name);	40✔
94	}
95
96	return compiledArgs;	46✔
97	}
98
99	_compileDeclarationFunction(decl, parentIRBuilder) {
100	const irBuilder = new JSIr.IRBuilder(parentIRBuilder ? parentIRBuilder.nextRegister : 0, ['__emit']);	20✔
101
102	const functionScope = new Scope(this._declarations);	20✔
103	const args = this._declareArguments(decl.args, functionScope, irBuilder);	20✔
104	const opCompiler = new OpCompiler(this, functionScope, irBuilder);	20✔
105	let op;
106
107	const hints = new QueryInvocationHints(new Set(getDefaultProjection(decl.schema)));	20✔
108
109	switch (decl.type) {	20✔
110	case 'query':
111	op = compileTableToOps(decl.value, [], hints);	6✔
112	opCompiler.compileQueryDeclaration(op);	6✔
113	break;	6✔
114
115	case 'stream':
116	op = compileStreamToOps(decl.value, hints);	2✔
117	opCompiler.compileStreamDeclaration(op);	2✔
118	break;	2✔
119
120	case 'action':
121	opCompiler.compileActionDeclaration(decl.value);	12✔
122	break;	12✔
123	}
124
125	let code;
126	let register;
127	if (parentIRBuilder) {	20✔
128	parentIRBuilder.skipRegisterRange(irBuilder.registerRange);	2✔
129
130	register = parentIRBuilder.allocRegister();	2✔
131	parentIRBuilder.add(new JSIr.AsyncFunctionDeclaration(register, irBuilder));	2✔
132	code = null;	2✔
133	} else {
134	register = null;	18✔
135	code = this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);	18✔
136	this._toplevelscope[decl.name] = code;	18✔
137	}
138
139	this._declarations.set(decl.name, {	20✔
140	type: 'declaration',
141	args, code, register,
142	schema: decl.schema,
143	});
144	}
145
146	_compileAssignment(assignment, irBuilder, { hasAnyStream, forProcedure}) {
147	const opCompiler = new OpCompiler(this, this._declarations, irBuilder, forProcedure);	18✔
148
149	// at the top level, assignments can be referred to by streams, so
150	// they need to be persistent (save to disk) such that when the program
151	// is restarted, the result can be reused.
152	// (this is only needed in top-level since stream is not allowed within
153	// procedures)
154	const isPersistent = hasAnyStream;	18✔
155
156	let register;
157
158	if (assignment.isAction) {	18✔
159	register = opCompiler.compileActionAssignment(assignment.value, isPersistent);	2✔
160	} else {
161	const hints = new QueryInvocationHints(new Set(getDefaultProjection(assignment.value.schema)));	16✔
162	const tableop = compileTableToOps(assignment.value, [], hints);	16✔
163	register = opCompiler.compileAssignment(tableop, isPersistent);	16✔
164	}
165
166	this._declarations.set(assignment.name, {	18✔
167	type: 'assignment',
168	isPersistent, register,
169	schema: assignment.schema
170	});
171	}
172
173	_compileProcedure(decl, parentIRBuilder) {
174	const saveScope = this._declarations;	26✔
175
176	const irBuilder = new JSIr.IRBuilder(parentIRBuilder ? parentIRBuilder.nextRegister : 0, ['__emit']);	26✔
177
178	const procid = this._nextProcId++;	26✔
179	irBuilder.setBeginEndHooks(	26✔
180	new JSIr.EnterProcedure(procid, decl.name),
181	new JSIr.ExitProcedure(procid, decl.name)
182	);
183
184	const procedureScope = new Scope(this._declarations);	26✔
185	const args = this._declareArguments(decl.args, procedureScope, irBuilder);	26✔
186	this._declarations = procedureScope;	26✔
187
188	this._compileInScope(decl.value, decl.value.rules, irBuilder, {	26✔
189	hasAnyStream: false,
190	forProcedure: true
191	});
192
193	let code;
194	let register;
195	if (parentIRBuilder) {	26✔
196	parentIRBuilder.skipRegisterRange(irBuilder.registerRange);	8✔
197
198	register = parentIRBuilder.allocRegister();	8✔
199	parentIRBuilder.add(new JSIr.AsyncFunctionDeclaration(register, irBuilder));	8✔
200	code = null;	8✔
201	} else {
202	register = null;	18✔
203	code = this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);	18✔
204	this._toplevelscope[decl.name] = code;	18✔
205	}
206
207	this._declarations = saveScope;	26✔
208
209	this._declarations.set(decl.name, {	26✔
210	type: 'procedure',
211	args, code, register,
212	schema: decl.schema,
213	});
214	}
215
216	_doCompileStatement(stmt, irBuilder, forProcedure) {
217	const opCompiler = new OpCompiler(this, this._declarations, irBuilder, forProcedure);	398✔
218	let ruleop = compileStatementToOp(stmt);	398✔
219	opCompiler.compileStatement(ruleop);	398✔
220	}
221
222	_compileRule(rule) {
223	// each rule goes into its own JS function
224	const irBuilder = new JSIr.IRBuilder();	120✔
225	this._doCompileStatement(rule, irBuilder);	120✔
226
227	return this._testMode ? irBuilder.codegen() : irBuilder.compile(this._toplevelscope);	120✔
228	}
229
230	_compileInScope(program, stmts, irBuilder, { hasAnyStream, forProcedure }) {
231	for (let decl of program.declarations) {	346✔
232	if (['stream', 'query', 'action'].indexOf(decl.type) >= 0)	46✔
233	this._declarations[decl.name] = this._compileDeclarationFunction(decl, irBuilder);	20✔
234	else if (decl.type === 'procedure')	26!
235	this._compileProcedure(decl, irBuilder);	26✔
236	else
237	throw new NotImplementedError(decl);	×
238	}
239
240	if (stmts.length === 0)	346✔
241	return null;	108✔
242
243	// all immediate statements are compiled into a single function, so we
244	// create a single irBuilder that we share
245	if (!irBuilder) {	238✔
246	irBuilder = new JSIr.IRBuilder();	212✔
247	const procid = this._nextProcId++;	212✔
248	irBuilder.setBeginEndHooks(	212✔
249	new JSIr.EnterProcedure(procid),
250	new JSIr.ExitProcedure(procid)
251	);
252	}
253
254	for (let i = 0; i < stmts.length; i++) {	238✔
255	// if this is not the first statement, clear the get cache before running it
256	if (i !== 0)	296✔
257	irBuilder.add(new JSIr.ClearGetCache());	58✔
258	if (stmts[i].isAssignment)	296✔
259	this._compileAssignment(stmts[i], irBuilder, { hasAnyStream, forProcedure });	18✔
260	else
261	this._doCompileStatement(stmts[i], irBuilder, forProcedure);	278✔
262	}
263
264	return irBuilder;	238✔
265	}
266
267	_verifyCompilable(program) {
268	if (program.principal !== null)	320!
UNCOV 269	throw new NotCompilableError(`Remote programs cannot be compiled, they must be sent to a different runtime instead`);	×
270
271	for (let slot of program.iterateSlots2()) {	320✔
272	if (slot instanceof Ast.Selector) {	1,088✔
273	if (slot.principal !== null)	450!
UNCOV 274	throw new NotCompilableError(`Remote primitives cannot be compiled, they must be lowered and sent to a different runtime instead`);	×
275	continue;	450✔
276	}
277	if (!slot.isCompilable())	638!
UNCOV 278	throw new NotCompilableError(`Programs with slots or unresolved values cannot be compiled, and must be slot-filled first`);	×
279	}
280	}
281
282	async compileProgram(program) {
283	await typeCheckProgram(program, this._schemaRetriever);	320✔
284	this._verifyCompilable(program);	320✔
285
286	const compiledRules = [];	320✔
287	const immediate = [];	320✔
288	const rules = [];	320✔
289
290	for (let stmt of program.rules) {	320✔
291	if (stmt.isAssignment \|\| stmt.isCommand)	370✔
292	immediate.push(stmt);	250✔
293	else
294	rules.push(stmt);	120✔
295	}
296
297	this._declarations = new Scope;	320✔
298	const commandIRBuilder = this._compileInScope(program, immediate, null, {	320✔
299	hasAnyStream: rules.length > 0,
300	forProcedure: false
301	});
302	let compiledCommand;
303	if (commandIRBuilder !== null) {	320✔
304	if (this._testMode)	212✔
305	compiledCommand = commandIRBuilder.codegen();	112✔
306	else
307	compiledCommand = commandIRBuilder.compile(this._toplevelscope, this._astVars);	100✔
308	} else {
309	compiledCommand = null;	108✔
310	}
311	for (let rule of rules)	320✔
312	compiledRules.push(this._compileRule(rule));	120✔
313
314	return new CompiledProgram(this._nextStateVar, compiledCommand, compiledRules);	320✔
315	}
316	};

stanford-oval / thingtalk / 986

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous