1104

pending completion

Build # 1104

Build Type

Pull #254

travis-ci-com

Committed by

web-flow

Commit Message

Merge adc5a4219 into 65b4e469a

Pull Request Pull Request #254: Making the Spotify skill work

Run Details

6586 of 12391 branches covered (53.15%)

Branch coverage included in aggregate %.

122 of 155 new or added lines in 12 files covered. (78.71%)

39 existing lines in 5 files now uncovered.

15161 of 23969 relevant lines covered (63.25%)

2147935.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.2

/languages/thingtalk/utils.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//           2019 National Taiwan University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//         Elvis Yu-Jing Lin <r06922068@ntu.edu.tw> <elvisyjlin@gmail.com>
"use strict";

const assert = require('assert');
const ThingTalk = require('thingtalk');
const Ast = ThingTalk.Ast;
const Type = ThingTalk.Type;

function typeToStringSafe(type) {
    if (type.isArray)
        return 'Array__' + typeToStringSafe(type.elem);
    else if (type.isEntity)
        return 'Entity__' + type.type.replace(':', '__');
    else if (type.isMeasure)
        return 'Measure_' + type.unit;
    else if (type.isEnum)
        return 'Enum__' + type.entries.join('__');
    else
        return String(type);
}

function clean(name) {
    if (/^[vwgp]_/.test(name))
        name = name.substr(2);
    return name.replace(/_/g, ' ').replace(/([^A-Z ])([A-Z])/g, '$1 $2').toLowerCase();
}


function makeFilter(loader, pname, op, value, negate = false) {
    assert(pname instanceof Ast.Value.VarRef);
    let vtype = value.getType();
    let ptype = vtype;
    if (ptype.isEntity && ptype.type === 'tt:url')
        return null;
    if (op === 'contains') {
        ptype = Type.Array(vtype);
        if (vtype.isString)
            op = 'contains~';
    } else if (op === '==' && vtype.isString) {
        op = '=~';
    }
    if (!loader.params.out.has(pname.name + '+' + ptype) && pname.name !== 'id')
        return null;
    if (loader.flags.turking && value.isEnum)
        return null;

    let f = new Ast.BooleanExpression.Atom(null, pname.name, op, value);
    if (negate)
        return new Ast.BooleanExpression.Not(null, f);
    else
        return f;
}

function makeAndFilter(loader, param, op, values, negate=false) {
    if (values.length !== 2)
        return null;
    if (values[0].name === values[1].name)
        return null;
    const operands  = values.map((v) => makeFilter(loader, param, op, v));
    if (operands.includes(null))
        return null;
    const f = new Ast.BooleanExpression.And(null, operands);
    if (negate)
        return new Ast.BooleanExpression.Not(null, f);
    return f;
}

function isHumanEntity(type) {
    if (type.isEntity)
        return isHumanEntity(type.type);
    if (type.isArray)
        return isHumanEntity(type.elem);
    if (typeof type !== 'string')
        return false;
    if (['tt:contact', 'tt:username', 'org.wikidata:human'].includes(type))
        return true;
    if (type.startsWith('org.schema') && type.endsWith(':Person'))
        return true;
    return false;
}

const PARAM_REGEX = /\$(?:\$|([a-zA-Z0-9_]+(?![a-zA-Z0-9_]))|{([a-zA-Z0-9_]+)(?::([a-zA-Z0-9_-]+))?})/;

function* split(pattern, regexp) {
    // a split that preserves capturing parenthesis

    let clone = new RegExp(regexp, 'g');
    let match = clone.exec(pattern);

    let i = 0;
    while (match !== null) {
        if (match.index > i)
            yield pattern.substring(i, match.index);
        yield match;
        i = clone.lastIndex;
        match = clone.exec(pattern);
    }
    if (i < pattern.length)
        yield pattern.substring(i, pattern.length);
}

function splitParams(utterance) {
    return Array.from(split(utterance, PARAM_REGEX));
}

function tokenizeExample(tokenizer, utterance, id) {
    let replaced = '';
    let params = [];

    for (let chunk of splitParams(utterance.trim())) {
        if (chunk === '')
            continue;
        if (typeof chunk === 'string') {
            replaced += chunk;
            continue;
        }

        let [match, param1, param2, opt] = chunk;
        if (match === '$$') {
            replaced += '$';
            continue;
        }
        let param = param1 || param2;
        replaced += ' ____ ';
        params.push([param, opt]);
    }

    const tokenized = tokenizer.tokenize(replaced);
    const tokens = tokenized.tokens;
    const entities = tokenized.entities;

    if (Object.keys(entities).length > 0)
        throw new Error(`Error in Example ${id}: Cannot have entities in the utterance`);

    let preprocessed = '';
    let first = true;
    for (let token of tokens) {
        if (token === '____') {
            let [param, opt] = params.shift();
            if (opt)
                token = '${' + param + ':' + opt + '}';
            else
                token = '${' + param + '}';
        } else if (token === '$') {
            token = '$$';
        }
        if (!first)
            preprocessed += ' ';
        preprocessed += token;
        first = false;
    }

    return preprocessed;
}

function isSameFunction(fndef1, fndef2) {
    if (!fndef1.class || !fndef2.class) // a join
        return false;
    return fndef1.class.name === fndef2.class.name &&
        fndef1.name === fndef2.name;
}

function isExecutable(stmt) {
    let hasUndefined = false;
    const visitor = new class extends Ast.NodeVisitor {
        visitInvocation(invocation) {
            const requireEither = invocation.schema.getAnnotation('require_either');
            if (requireEither) {
                const params = new Set;
                for (let in_param of invocation.in_params)
                    params.add(in_param.name);

                for (let requirement of requireEither) {
                    let satisfied = false;
                    for (let option of requirement) {
                        if (params.has(option)) {
                            satisfied = true;
                            break;
                        }
                    }
                    if (!satisfied)
                        hasUndefined = true;
                }
            }

            return true;
        }

        visitValue(value) {
            if (value.isUndefined)
                hasUndefined = true;
            return true;
        }
    };
    stmt.visit(visitor);
    return !hasUndefined;
}

/**
 * Normalize the #[confirm] annotation.
 *
 * #[confirm] is a three-state enum annotation with values:
 * - #[confirm=enum(confirm)]: must confirm explicitly with all parameters before the
 *   function is called (using a statement with #[confirm=enum(confirmed)] annotation)
 * - #[confirm=enum(display_result)]: the result of any query that feeds into the parameters
 *   of this function should be displayed before the function is executed; this is encoded
 *   by splitting any compound statement into two statements, executed sequentially
 * - #[confirm=enum(auto)]: the function can be called without explicit confirmation, even
 *   if some of the parameters are coming from other functions; this is the only #[confirm]
 *   that allows the function to be called multiple times in a single statement
 *
 * For legacy/ease of development reasons, if unspecified #[confirm] defaults to "confirm"
 * for actions (full confirmation before executing side effects) and "display_result" for
 * queries (splitting table joins into two statements).
 *
 * Also, #[confirm] can be specified as a boolean: "true" means "confirm" and "false" means
 * "display_result".
 */
function normalizeConfirmAnnotation(fndef) {
    const value = fndef.getAnnotation('confirm');
    if (value === undefined) // unspecified
        return fndef.functionType === 'action' ? 'confirm' : 'display_result';

    if (typeof value === 'boolean')
        return value ? 'confirm' : 'display_result';

    return value;
}

module.exports = {
    clean,

    split,
    splitParams,
    tokenizeExample,

    isExecutable,
    normalizeConfirmAnnotation,

    isUnaryTableToTableOp(table) {
        return table.isFilter ||
            table.isProjection ||
            table.isCompute ||
            table.isAlias ||
            table.isSort ||
            table.isIndex ||
            table.isSlice ||
            table.isAggregation ||
            table.isSequence ||
            table.isHistory;
    },
    isUnaryStreamToTableOp(table) {
        return table.isWindow || table.isTimeSeries;
    },
    isUnaryStreamToStreamOp(stream) {
        return stream.isEdgeNew ||
            stream.isEdgeFilter ||
            stream.isFilter ||
            stream.isProjection ||
            stream.isCompute ||
            stream.isAlias;
    },
    isUnaryTableToStreamOp(stream) {
        return stream.isMonitor;
    },
    isSameFunction,

    typeToStringSafe,
    makeFilter,
    makeAndFilter,

    isHumanEntity
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	// 2019 National Taiwan University
7	//
8	// Licensed under the Apache License, Version 2.0 (the "License");
9	// you may not use this file except in compliance with the License.
10	// You may obtain a copy of the License at
11	//
12	// http://www.apache.org/licenses/LICENSE-2.0
13	//
14	// Unless required by applicable law or agreed to in writing, software
15	// distributed under the License is distributed on an "AS IS" BASIS,
16	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17	// See the License for the specific language governing permissions and
18	// limitations under the License.
19	//
20	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
21	// Elvis Yu-Jing Lin <r06922068@ntu.edu.tw> <elvisyjlin@gmail.com>
22	"use strict";
23
24	const assert = require('assert');	57✔
25	const ThingTalk = require('thingtalk');	57✔
26	const Ast = ThingTalk.Ast;	57✔
27	const Type = ThingTalk.Type;	57✔
28
29	function typeToStringSafe(type) {
30	if (type.isArray)	205,758✔
31	return 'Array__' + typeToStringSafe(type.elem);	6,326✔
32	else if (type.isEntity)	199,432✔
33	return 'Entity__' + type.type.replace(':', '__');	84,036✔
34	else if (type.isMeasure)	115,396✔
35	return 'Measure_' + type.unit;	4,146✔
36	else if (type.isEnum)	111,250✔
37	return 'Enum__' + type.entries.join('__');	23,722✔
38	else
39	return String(type);	87,528✔
40	}
41
42	function clean(name) {
43	if (/^[vwgp]_/.test(name))	2,558✔
44	name = name.substr(2);	1,231✔
45	return name.replace(/_/g, ' ').replace(/([^A-Z ])([A-Z])/g, '$1 $2').toLowerCase();	2,558✔
46	}
47
48
49	function makeFilter(loader, pname, op, value, negate = false) {	1,419,976✔
50	assert(pname instanceof Ast.Value.VarRef);	3,864,291✔
51	let vtype = value.getType();	3,864,291✔
52	let ptype = vtype;	3,864,291✔
53	if (ptype.isEntity && ptype.type === 'tt:url')	3,864,291✔
54	return null;	61,617✔
55	if (op === 'contains') {	3,802,674✔
56	ptype = Type.Array(vtype);	2,657,241✔
57	if (vtype.isString)	2,657,241✔
58	op = 'contains~';	76,027✔
59	} else if (op === '==' && vtype.isString) {	1,145,433✔
60	op = '=~';	24,048✔
61	}
62	if (!loader.params.out.has(pname.name + '+' + ptype) && pname.name !== 'id')	3,802,674✔
63	return null;	3,424,038✔
64	if (loader.flags.turking && value.isEnum)	378,636!
65	return null;	×
66
67	let f = new Ast.BooleanExpression.Atom(null, pname.name, op, value);	378,636✔
68	if (negate)	378,636✔
69	return new Ast.BooleanExpression.Not(null, f);	11,595✔
70	else
71	return f;	367,041✔
72	}
73
74	function makeAndFilter(loader, param, op, values, negate=false) {	×
75	if (values.length !== 2)	7,278,513!
76	return null;	×
77	if (values[0].name === values[1].name)	7,278,513✔
78	return null;	6,568,525✔
79	const operands = values.map((v) => makeFilter(loader, param, op, v));	1,419,976✔
80	if (operands.includes(null))	709,988✔
81	return null;	694,454✔
82	const f = new Ast.BooleanExpression.And(null, operands);	15,534✔
83	if (negate)	15,534✔
84	return new Ast.BooleanExpression.Not(null, f);	3,400✔
85	return f;	12,134✔
86	}
87
88	function isHumanEntity(type) {
89	if (type.isEntity)	4,136✔
90	return isHumanEntity(type.type);	719✔
91	if (type.isArray)	3,417✔
92	return isHumanEntity(type.elem);	523✔
93	if (typeof type !== 'string')	2,894✔
94	return false;	2,175✔
95	if (['tt:contact', 'tt:username', 'org.wikidata:human'].includes(type))	719✔
96	return true;	1✔
97	if (type.startsWith('org.schema') && type.endsWith(':Person'))	718✔
98	return true;	58✔
99	return false;	660✔
100	}
101
102	const PARAM_REGEX = /\$(?:\$\|([a-zA-Z0-9_]+(?![a-zA-Z0-9_]))\|{([a-zA-Z0-9_]+)(?::([a-zA-Z0-9_-]+))?})/;	57✔
103
104	function* split(pattern, regexp) {
105	// a split that preserves capturing parenthesis
106
107	let clone = new RegExp(regexp, 'g');	84✔
108	let match = clone.exec(pattern);	84✔
109
110	let i = 0;	84✔
111	while (match !== null) {	84✔
112	if (match.index > i)	69!
113	yield pattern.substring(i, match.index);	69✔
114	yield match;	69✔
115	i = clone.lastIndex;	69✔
116	match = clone.exec(pattern);	69✔
117	}
118	if (i < pattern.length)	84✔
119	yield pattern.substring(i, pattern.length);	46✔
120	}
121
122	function splitParams(utterance) {
123	return Array.from(split(utterance, PARAM_REGEX));	84✔
124	}
125
126	function tokenizeExample(tokenizer, utterance, id) {
127	let replaced = '';	84✔
128	let params = [];	84✔
129
130	for (let chunk of splitParams(utterance.trim())) {	84✔
131	if (chunk === '')	184!
132	continue;	×
133	if (typeof chunk === 'string') {	184✔
134	replaced += chunk;	115✔
135	continue;	115✔
136	}
137
138	let [match, param1, param2, opt] = chunk;	69✔
139	if (match === '$$') {	69!
140	replaced += '$';	×
141	continue;	×
142	}
143	let param = param1 \|\| param2;	69✔
144	replaced += ' ____ ';	69✔
145	params.push([param, opt]);	69✔
146	}
147
148	const tokenized = tokenizer.tokenize(replaced);	84✔
149	const tokens = tokenized.tokens;	84✔
150	const entities = tokenized.entities;	84✔
151
152	if (Object.keys(entities).length > 0)	84!
153	throw new Error(`Error in Example ${id}: Cannot have entities in the utterance`);	×
154
155	let preprocessed = '';	84✔
156	let first = true;	84✔
157	for (let token of tokens) {	84✔
158	if (token === '____') {	501✔
159	let [param, opt] = params.shift();	69✔
160	if (opt)	69✔
161	token = '${' + param + ':' + opt + '}';	7✔
162	else
163	token = '${' + param + '}';	62✔
164	} else if (token === '$') {	432!
165	token = '$$';	×
166	}
167	if (!first)	501✔
168	preprocessed += ' ';	417✔
169	preprocessed += token;	501✔
170	first = false;	501✔
171	}
172
173	return preprocessed;	84✔
174	}
175
176	function isSameFunction(fndef1, fndef2) {
177	if (!fndef1.class \|\| !fndef2.class) // a join	6,934,115!
178	return false;	×
179	return fndef1.class.name === fndef2.class.name &&	6,934,115✔
180	fndef1.name === fndef2.name;
181	}
182
183	function isExecutable(stmt) {
184	let hasUndefined = false;	19,250✔
185	const visitor = new class extends Ast.NodeVisitor {	19,250✔
186	visitInvocation(invocation) {
187	const requireEither = invocation.schema.getAnnotation('require_either');	19,250✔
188	if (requireEither) {	19,250✔
189	const params = new Set;	1,615✔
190	for (let in_param of invocation.in_params)	1,615✔
191	params.add(in_param.name);	4,848✔
192
193	for (let requirement of requireEither) {	1,615✔
194	let satisfied = false;	1,615✔
195	for (let option of requirement) {	1,615✔
196	if (params.has(option)) {	2,481✔
197	satisfied = true;	1,118✔
198	break;	1,118✔
199	}
200	}
201	if (!satisfied)	1,615✔
202	hasUndefined = true;	497✔
203	}
204	}
205
206	return true;	19,250✔
207	}
208
209	visitValue(value) {
210	if (value.isUndefined)	57,378✔
211	hasUndefined = true;	24,201✔
212	return true;	57,378✔
213	}
214	};
215	stmt.visit(visitor);	19,250✔
216	return !hasUndefined;	19,250✔
217	}
218
219	/**
220	* Normalize the #[confirm] annotation.
221	*
222	* #[confirm] is a three-state enum annotation with values:
223	* - #[confirm=enum(confirm)]: must confirm explicitly with all parameters before the
224	* function is called (using a statement with #[confirm=enum(confirmed)] annotation)
225	* - #[confirm=enum(display_result)]: the result of any query that feeds into the parameters
226	* of this function should be displayed before the function is executed; this is encoded
227	* by splitting any compound statement into two statements, executed sequentially
228	* - #[confirm=enum(auto)]: the function can be called without explicit confirmation, even
229	* if some of the parameters are coming from other functions; this is the only #[confirm]
230	* that allows the function to be called multiple times in a single statement
231	*
232	* For legacy/ease of development reasons, if unspecified #[confirm] defaults to "confirm"
233	* for actions (full confirmation before executing side effects) and "display_result" for
234	* queries (splitting table joins into two statements).
235	*
236	* Also, #[confirm] can be specified as a boolean: "true" means "confirm" and "false" means
237	* "display_result".
238	*/
239	function normalizeConfirmAnnotation(fndef) {
240	const value = fndef.getAnnotation('confirm');	38,268✔
241	if (value === undefined) // unspecified	38,268✔
242	return fndef.functionType === 'action' ? 'confirm' : 'display_result';	1!
243
244	if (typeof value === 'boolean')	38,267!
245	return value ? 'confirm' : 'display_result';	38,267✔
246
NEW 247	return value;	×
248	}
249
250	module.exports = {	57✔
251	clean,
252
253	split,
254	splitParams,
255	tokenizeExample,
256
257	isExecutable,
258	normalizeConfirmAnnotation,
259
260	isUnaryTableToTableOp(table) {
261	return table.isFilter \|\|	×
262	table.isProjection \|\|
263	table.isCompute \|\|
264	table.isAlias \|\|
265	table.isSort \|\|
266	table.isIndex \|\|
267	table.isSlice \|\|
268	table.isAggregation \|\|
269	table.isSequence \|\|
270	table.isHistory;
271	},
272	isUnaryStreamToTableOp(table) {
273	return table.isWindow \|\| table.isTimeSeries;	×
274	},
275	isUnaryStreamToStreamOp(stream) {
276	return stream.isEdgeNew \|\|	×
277	stream.isEdgeFilter \|\|
278	stream.isFilter \|\|
279	stream.isProjection \|\|
280	stream.isCompute \|\|
281	stream.isAlias;
282	},
283	isUnaryTableToStreamOp(stream) {
284	return stream.isMonitor;	×
285	},
286	isSameFunction,
287
288	typeToStringSafe,
289	makeFilter,
290	makeAndFilter,
291
292	isHumanEntity
293	};

stanford-oval / genie-toolkit / 1104

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous