stanford-oval / genie-toolkit / 645

pending completion

Build # 645

Build Type

Pull #137

travis-ci-com

Committed by

web-flow

Commit Message

thingtalk: switch how computation templates are defined

We need to make sure the non-terminals are defined even if the
turking flag is set, or generation will fail.

Pull Request Pull Request #137: Second round of dialogue templates

Run Details

1729 of 4536 branches covered (38.12%)

Branch coverage included in aggregate %.

36 of 96 new or added lines in 14 files covered. (37.5%)

38 existing lines in 5 files now uncovered.

4346 of 9203 relevant lines covered (47.22%)

583148.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.53

/lib/utils.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2015-2019 The Board of Trustees of the Leland Stanford Junior University
//           2019 National Taiwan University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//         Elvis Yu-Jing Lin <r06922068@ntu.edu.tw> <elvisyjlin@gmail.com>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const Inflectors = require('en-inflectors').Inflectors;
const Tag = require('en-pos').Tag;

const MAX_SMALL_INTEGER = 12;

class ValidationError extends Error {
    constructor(message) {
        super(message);
        this.code = 'EINVAL';
    }
}

function clean(name) {
    if (/^[vwgp]_/.test(name))
        name = name.substr(2);
    return name.replace(/_/g, ' ').replace(/([^A-Z ])([A-Z])/g, '$1 $2').toLowerCase();
}

function pluralize(name) {
    if (!name.includes(' ')) {
        if (new Tag([name]).initial().tags[0] === 'NN')
            return new Inflectors(name).toPlural();
        return name;
    } else {
        const words = name.split(' ');
        const tags = new Tag(words).initial().tags;
        if (tags[tags.length - 1] !== 'NN')
            return name;
        else if (['VB', 'VBP', 'VBZ', 'VBD'].includes(tags[0]))
            return name;
        words[words.length - 1] = pluralize(words[words.length - 1]);
        return words.join(' ');
    }
}

const PARAM_REGEX = /\$(?:\$|([a-zA-Z0-9_]+(?![a-zA-Z0-9_]))|{([a-zA-Z0-9_]+)(?::([a-zA-Z0-9_-]+))?})/;

function* split(pattern, regexp) {
    // a split that preserves capturing parenthesis

    let clone = new RegExp(regexp, 'g');
    let match = clone.exec(pattern);

    let i = 0;
    while (match !== null) {
        if (match.index > i)
            yield pattern.substring(i, match.index);
        yield match;
        i = clone.lastIndex;
        match = clone.exec(pattern);
    }
    if (i < pattern.length)
        yield pattern.substring(i, pattern.length);
}

function namedString(key, idx) {
    return `str:${key}::${idx}:`;
}

const ENTITIES = {
    DURATION: (idx) => ({ value: idx + 2, unit: 'ms' }),
    NUMBER: (idx) => idx + MAX_SMALL_INTEGER + 1,
    DATE: (idx) => ({ day: idx + 2, month: 1, year: 2018 }),
    TIME: (idx) => ({ hour: Math.floor(idx/4), minute: [0, 15, 30, 45][idx % 4], second: 0 }),
    CURRENCY: (idx) => ({ value: idx + 2, unit: 'usd' }),
    LOCATION: (idx) => ({ latitude: idx + 2, longitude: idx + 2 }),
    QUOTED_STRING: (idx) => namedString('QUOTED_STRING', idx),
    PATH_NAME: (idx) => namedString('PATH_NAME', idx),
    URL: (idx) => namedString('URL', idx),
    PHONE_NUMBER: (idx) => namedString('PHONE_NUMBER', idx),
    EMAIL_ADDRESS: (idx) => namedString('EMAIL_ADDRESS', idx),
    USERNAME: (idx) => namedString('USERNAME', idx),
    HASHTAG: (idx) => namedString('HASHTAG', idx),
};
Object.freeze(ENTITIES);

function makeDummyMeasure(entity) {
    const match = /^MEASURE_([A-Za-z]+)_([0-9]+)$/.exec(entity);
    return { unit: match[1], value: 2.0 + parseInt(match[2]) };
}

function parseDate(form) {
    if (form instanceof Date)
        return form;

    let now = new Date;
    let year = form.year;
    if (year < 0 || year === undefined)
        year = now.getFullYear();
    let month = form.month;
    if (month < 0 || month === undefined)
        month = now.getMonth() + 1;
    let day = form.day;
    if (day < 0 || day === undefined)
        day = now.getDate();
    let hour = form.hour;
    if (hour < 0 || hour === undefined)
        hour = 0;
    let minute = form.minute;
    if (minute < 0 || minute === undefined)
        minute = 0;
    let second = form.second;
    if (second < 0 || second === undefined)
        second = 0;
    let millisecond = (second - Math.floor(second))*1000;
    second = Math.floor(second);

    return new Date(year, month-1, day, hour, minute, second, millisecond);
}

function entitiesEqual(type, one, two) {
    if (one === two)
        return true;
    if (!one || !two)
        return false;
    if (type.startsWith('GENERIC_ENTITY_'))
        return (one.value === two.value);

    if (type.startsWith('MEASURE_') ||
        type === 'DURATION')
        return one.value === two.value && one.unit === two.unit;

    switch (type) {
    case 'CURRENCY':
        return one.value === two.value && one.unit === two.unit;
    case 'TIME':
        return one.hour === two.hour &&
            one.minute === two.minute &&
            (one.second || 0) === (two.second || 0);
    case 'DATE':
        if (!(one instanceof Date))
            one = parseDate(one);
        if (!(two instanceof Date))
            two = parseDate(two);

        return +one === +two;
    case 'LOCATION':
        return Math.abs(one.latitude - two.latitude) < 0.01 &&
            Math.abs(one.longitude - two.longitude) < 0.01;
    }

    return false;
}

function splitParams(utterance) {
    return Array.from(split(utterance, PARAM_REGEX));
}

async function tokenizeExample(tokenizer, utterance, id, language) {
    let replaced = '';
    let params = [];

    for (let chunk of splitParams(utterance.trim())) {
        if (chunk === '')
            continue;
        if (typeof chunk === 'string') {
            replaced += chunk;
            continue;
        }

        let [match, param1, param2, opt] = chunk;
        if (match === '$$') {
            replaced += '$';
            continue;
        }
        let param = param1 || param2;
        replaced += ' ____ ';
        params.push([param, opt]);
    }

    const tokenized = await tokenizer.tokenize(language, replaced);
    const tokens = tokenized.tokens;
    const entities = tokenized.entities;

    if (Object.keys(entities).length > 0)
        throw new ValidationError(`Error in Example ${id}: Cannot have entities in the utterance`);

    let preprocessed = '';
    let first = true;
    for (let token of tokens) {
        if (token === '____') {
            let [param, opt] = params.shift();
            if (opt)
                token = '${' + param + ':' + opt + '}';
            else
                token = '${' + param + '}';
        } else if (token === '$') {
            token = '$$';
        }
        if (!first)
            preprocessed += ' ';
        preprocessed += token;
        first = false;
    }

    return preprocessed;
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_([0-9]+)$/;

function parseEntity(token) {
    const match = ENTITY_MATCH_REGEX.exec(token);
    if (match !== null)
        return [match[1], parseInt(match[2], 10)];
    else
        return null;
}

function makeDummyEntity(token) {
    const match = ENTITY_MATCH_REGEX.exec(token);
    if (match === null)
        throw new TypeError(`invalid entity ${token}`);
    const [,entityType,entityIndex] = match;
    assert(!Number.isNaN(Number(entityIndex)), token);

    if (entityType.startsWith('MEASURE_'))
        return makeDummyMeasure(token);
    else if (entityType.startsWith('GENERIC_ENTITY_'))
        return { value: namedString(entityType.substring('GENERIC_'.length), entityIndex), display: null };
    else if (!(entityType in ENTITIES))
        throw new Error(`missing entity ${token}`);
    else
        return ENTITIES[entityType](Number(entityIndex));
}

module.exports = {
    splitParams,
    split,
    clean,
    pluralize,
    tokenizeExample,

    namedString,
    parseEntity,
    makeDummyEntity,
    makeDummyEntities(preprocessed) {
        const entities = {};
        for (let token of preprocessed.split(' ')) {
            if (/^[A-Z]/.test(token))
                entities[token] = makeDummyEntity(token);
        }
        return entities;
    },

    renumberEntities(tokenized, context) {
        const { entities, tokens } = tokenized;
        const out = {};

        const offsets = {};
        for (let key in context) {
            const [, type, num] = /^(.+)_([0-9]+)$/.exec(key);

            let next = offsets[type] || 0;
            offsets[type] = Math.max(next, parseInt(num, 10) + 1);
            out[key] = context[key];
        }

        const rewrites = {};

        for (let i = 0; i < tokens.length; i++) {
            if (tokens[i] in rewrites) {
                tokens[i] = rewrites[tokens[i]];
                continue;
            }

            if (tokens[i] in entities) {
                const key = tokens[i];
                const [, type, ] = /^(.+)_([0-9]+)$/.exec(key);
                const value = entities[key];

                let found = false;
                for (let what in context) {
                    if (!what.startsWith(type + '_'))
                        continue;

                    if (entitiesEqual(type, context[what], value)) {
                        found = true;
                        rewrites[key] = what;
                        tokens[i] = what;
                        break;
                    }
                }

                if (!found) {
                    let next = offsets[type] || 0;
                    offsets[type] = next + 1;
                    const newkey = type + '_' + next;
                    out[newkey] = entities[key];
                    rewrites[key] = newkey;
                    tokens[i] = newkey;
                }
            }
        }

        tokenized.entities = out;
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2015-2019 The Board of Trustees of the Leland Stanford Junior University
6	// 2019 National Taiwan University
7	//
8	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
9	// Elvis Yu-Jing Lin <r06922068@ntu.edu.tw> <elvisyjlin@gmail.com>
10	//
11	// See COPYING for details
12	"use strict";
13
14	const assert = require('assert');	22✔
15	const Inflectors = require('en-inflectors').Inflectors;	22✔
16	const Tag = require('en-pos').Tag;	22✔
17
18	const MAX_SMALL_INTEGER = 12;	22✔
19
20	class ValidationError extends Error {
21	constructor(message) {
22	super(message);	×
23	this.code = 'EINVAL';	×
24	}
25	}
26
27	function clean(name) {
28	if (/^[vwgp]_/.test(name))	×
29	name = name.substr(2);	×
30	return name.replace(/_/g, ' ').replace(/([^A-Z ])([A-Z])/g, '$1 $2').toLowerCase();	×
31	}
32
33	function pluralize(name) {
34	if (!name.includes(' ')) {	×
35	if (new Tag([name]).initial().tags[0] === 'NN')	×
36	return new Inflectors(name).toPlural();	×
37	return name;	×
38	} else {
39	const words = name.split(' ');	×
40	const tags = new Tag(words).initial().tags;	×
41	if (tags[tags.length - 1] !== 'NN')	×
42	return name;	×
43	else if (['VB', 'VBP', 'VBZ', 'VBD'].includes(tags[0]))	×
44	return name;	×
45	words[words.length - 1] = pluralize(words[words.length - 1]);	×
46	return words.join(' ');	×
47	}
48	}
49
50	const PARAM_REGEX = /\$(?:\$\|([a-zA-Z0-9_]+(?![a-zA-Z0-9_]))\|{([a-zA-Z0-9_]+)(?::([a-zA-Z0-9_-]+))?})/;	22✔
51
52	function* split(pattern, regexp) {
53	// a split that preserves capturing parenthesis
54
55	let clone = new RegExp(regexp, 'g');	×
56	let match = clone.exec(pattern);	×
57
58	let i = 0;	×
59	while (match !== null) {	×
60	if (match.index > i)	×
61	yield pattern.substring(i, match.index);	×
62	yield match;	×
63	i = clone.lastIndex;	×
64	match = clone.exec(pattern);	×
65	}
66	if (i < pattern.length)	×
67	yield pattern.substring(i, pattern.length);	×
68	}
69
70	function namedString(key, idx) {
71	return `str:${key}::${idx}:`;	154✔
72	}
73
74	const ENTITIES = {	22✔
75	DURATION: (idx) => ({ value: idx + 2, unit: 'ms' }),	14✔
76	NUMBER: (idx) => idx + MAX_SMALL_INTEGER + 1,	18✔
77	DATE: (idx) => ({ day: idx + 2, month: 1, year: 2018 }),	16✔
78	TIME: (idx) => ({ hour: Math.floor(idx/4), minute: [0, 15, 30, 45][idx % 4], second: 0 }),	15✔
79	CURRENCY: (idx) => ({ value: idx + 2, unit: 'usd' }),	×
80	LOCATION: (idx) => ({ latitude: idx + 2, longitude: idx + 2 }),	1✔
81	QUOTED_STRING: (idx) => namedString('QUOTED_STRING', idx),	90✔
82	PATH_NAME: (idx) => namedString('PATH_NAME', idx),	4✔
83	URL: (idx) => namedString('URL', idx),	×
84	PHONE_NUMBER: (idx) => namedString('PHONE_NUMBER', idx),	1✔
85	EMAIL_ADDRESS: (idx) => namedString('EMAIL_ADDRESS', idx),	×
86	USERNAME: (idx) => namedString('USERNAME', idx),	26✔
87	HASHTAG: (idx) => namedString('HASHTAG', idx),	8✔
88	};
89	Object.freeze(ENTITIES);	22✔
90
91	function makeDummyMeasure(entity) {
92	const match = /^MEASURE_([A-Za-z]+)_([0-9]+)$/.exec(entity);	×
93	return { unit: match[1], value: 2.0 + parseInt(match[2]) };	×
94	}
95
96	function parseDate(form) {
97	if (form instanceof Date)	×
98	return form;	×
99
100	let now = new Date;	×
101	let year = form.year;	×
102	if (year < 0 \|\| year === undefined)	×
103	year = now.getFullYear();	×
104	let month = form.month;	×
105	if (month < 0 \|\| month === undefined)	×
106	month = now.getMonth() + 1;	×
107	let day = form.day;	×
108	if (day < 0 \|\| day === undefined)	×
109	day = now.getDate();	×
110	let hour = form.hour;	×
111	if (hour < 0 \|\| hour === undefined)	×
112	hour = 0;	×
113	let minute = form.minute;	×
114	if (minute < 0 \|\| minute === undefined)	×
115	minute = 0;	×
116	let second = form.second;	×
117	if (second < 0 \|\| second === undefined)	×
118	second = 0;	×
119	let millisecond = (second - Math.floor(second))*1000;	×
120	second = Math.floor(second);	×
121
122	return new Date(year, month-1, day, hour, minute, second, millisecond);	×
123	}
124
125	function entitiesEqual(type, one, two) {
126	if (one === two)	×
127	return true;	×
128	if (!one \|\| !two)	×
129	return false;	×
130	if (type.startsWith('GENERIC_ENTITY_'))	×
131	return (one.value === two.value);	×
132
133	if (type.startsWith('MEASURE_') \|\|	×
134	type === 'DURATION')
135	return one.value === two.value && one.unit === two.unit;	×
136
137	switch (type) {	×
138	case 'CURRENCY':
139	return one.value === two.value && one.unit === two.unit;	×
140	case 'TIME':
141	return one.hour === two.hour &&	×
142	one.minute === two.minute &&
143	(one.second \|\| 0) === (two.second \|\| 0);	×
144	case 'DATE':
145	if (!(one instanceof Date))	×
146	one = parseDate(one);	×
147	if (!(two instanceof Date))	×
148	two = parseDate(two);	×
149
150	return +one === +two;	×
151	case 'LOCATION':
152	return Math.abs(one.latitude - two.latitude) < 0.01 &&	×
153	Math.abs(one.longitude - two.longitude) < 0.01;
154	}
155
156	return false;	×
157	}
158
159	function splitParams(utterance) {
160	return Array.from(split(utterance, PARAM_REGEX));	×
161	}
162
163	async function tokenizeExample(tokenizer, utterance, id, language) {
164	let replaced = '';	×
165	let params = [];	×
166
167	for (let chunk of splitParams(utterance.trim())) {	×
168	if (chunk === '')	×
169	continue;	×
170	if (typeof chunk === 'string') {	×
171	replaced += chunk;	×
172	continue;	×
173	}
174
175	let [match, param1, param2, opt] = chunk;	×
176	if (match === '$$') {	×
177	replaced += '$';	×
178	continue;	×
179	}
180	let param = param1 \|\| param2;	×
181	replaced += ' ____ ';	×
182	params.push([param, opt]);	×
183	}
184
185	const tokenized = await tokenizer.tokenize(language, replaced);	×
186	const tokens = tokenized.tokens;	×
187	const entities = tokenized.entities;	×
188
189	if (Object.keys(entities).length > 0)	×
190	throw new ValidationError(`Error in Example ${id}: Cannot have entities in the utterance`);	×
191
192	let preprocessed = '';	×
193	let first = true;	×
194	for (let token of tokens) {	×
195	if (token === '____') {	×
196	let [param, opt] = params.shift();	×
197	if (opt)	×
198	token = '${' + param + ':' + opt + '}';	×
199	else
200	token = '${' + param + '}';	×
201	} else if (token === '$') {	×
202	token = '$$';	×
203	}
204	if (!first)	×
205	preprocessed += ' ';	×
206	preprocessed += token;	×
207	first = false;	×
208	}
209
210	return preprocessed;	×
211	}
212
213	const ENTITY_MATCH_REGEX = /^([A-Z].*)_([0-9]+)$/;	22✔
214
215	function parseEntity(token) {
216	const match = ENTITY_MATCH_REGEX.exec(token);	×
217	if (match !== null)	×
218	return [match[1], parseInt(match[2], 10)];	×
219	else
220	return null;	×
221	}
222
223	function makeDummyEntity(token) {
224	const match = ENTITY_MATCH_REGEX.exec(token);	218✔
225	if (match === null)	218!
NEW 226	throw new TypeError(`invalid entity ${token}`);	×
227	const [,entityType,entityIndex] = match;	218✔
228	assert(!Number.isNaN(Number(entityIndex)), token);	218✔
229
230	if (entityType.startsWith('MEASURE_'))	218!
231	return makeDummyMeasure(token);	×
232	else if (entityType.startsWith('GENERIC_ENTITY_'))	218✔
233	return { value: namedString(entityType.substring('GENERIC_'.length), entityIndex), display: null };	25✔
234	else if (!(entityType in ENTITIES))	193!
235	throw new Error(`missing entity ${token}`);	×
236	else
237	return ENTITIES[entityType](Number(entityIndex));	193✔
238	}
239
240	module.exports = {	22✔
241	splitParams,
242	split,
243	clean,
244	pluralize,
245	tokenizeExample,
246
247	namedString,
248	parseEntity,
249	makeDummyEntity,
250	makeDummyEntities(preprocessed) {
251	const entities = {};	1,114✔
252	for (let token of preprocessed.split(' ')) {	1,114✔
253	if (/^[A-Z]/.test(token))	7,508✔
254	entities[token] = makeDummyEntity(token);	214✔
255	}
256	return entities;	1,114✔
257	},
258
259	renumberEntities(tokenized, context) {
260	const { entities, tokens } = tokenized;	×
261	const out = {};	×
262
263	const offsets = {};	×
264	for (let key in context) {	×
265	const [, type, num] = /^(.+)_([0-9]+)$/.exec(key);	×
266
267	let next = offsets[type] \|\| 0;	×
268	offsets[type] = Math.max(next, parseInt(num, 10) + 1);	×
269	out[key] = context[key];	×
270	}
271
272	const rewrites = {};	×
273
274	for (let i = 0; i < tokens.length; i++) {	×
275	if (tokens[i] in rewrites) {	×
276	tokens[i] = rewrites[tokens[i]];	×
277	continue;	×
278	}
279
280	if (tokens[i] in entities) {	×
281	const key = tokens[i];	×
282	const [, type, ] = /^(.+)_([0-9]+)$/.exec(key);	×
283	const value = entities[key];	×
284
285	let found = false;	×
286	for (let what in context) {	×
287	if (!what.startsWith(type + '_'))	×
288	continue;	×
289
290	if (entitiesEqual(type, context[what], value)) {	×
291	found = true;	×
292	rewrites[key] = what;	×
293	tokens[i] = what;	×
294	break;	×
295	}
296	}
297
298	if (!found) {	×
299	let next = offsets[type] \|\| 0;	×
300	offsets[type] = next + 1;	×
301	const newkey = type + '_' + next;	×
302	out[newkey] = entities[key];	×
303	rewrites[key] = newkey;	×
304	tokens[i] = newkey;	×
305	}
306	}
307	}
308
309	tokenized.entities = out;	×
310	}
311	};

stanford-oval / genie-toolkit / 645

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous