1493

pending completion

Build # 1493

Build Type

Pull #405

travis-ci-com

Committed by

web-flow

Commit Message

Merge a226d5e7e into 17634db4b

Pull Request Pull Request #405: Small changes to improve synthesis for Thingpedia

Run Details

9755 of 22987 branches covered (42.44%)

Branch coverage included in aggregate %.

26 of 57 new or added lines in 5 files covered. (45.61%)

179 existing lines in 6 files now uncovered.

23540 of 42701 relevant lines covered (55.13%)

828749.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.33

/lib/utils/entity-utils.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import { Syntax } from 'thingtalk';

export type AnyEntity = Syntax.AnyEntity;
export type EntityMap = Syntax.EntityMap;

const MAX_SMALL_INTEGER = 12;

const ENTITIES : { [key : string] : (idx : number) => AnyEntity } = {
    DURATION: (idx) => ({ value: idx + 2, unit: 'ms' }),
    NUMBER: (idx) => idx + MAX_SMALL_INTEGER + 1,
    DATE: (idx) => ({ day: idx + 2, month: 1, year: 2018 }),
    TIME: (idx) => ({ hour: Math.floor(idx/4), minute: [0, 15, 30, 45][idx % 4], second: 0 }),
    CURRENCY: (idx) => ({ value: idx + 2, unit: 'usd' }),
    LOCATION: (idx) => ({ latitude: idx + 2, longitude: idx + 2 }),
    QUOTED_STRING: (idx) => namedString('QUOTED_STRING', idx),
    PATH_NAME: (idx) => namedString('PATH_NAME', idx),
    URL: (idx) => namedString('URL', idx),
    PHONE_NUMBER: (idx) => namedString('PHONE_NUMBER', idx),
    EMAIL_ADDRESS: (idx) => namedString('EMAIL_ADDRESS', idx),
    USERNAME: (idx) => namedString('USERNAME', idx),
    HASHTAG: (idx) => namedString('HASHTAG', idx),
    PICTURE: (idx) => namedString('PICTURE', idx)
};
Object.freeze(ENTITIES);

function namedString(key : string, idx : number) : string {
    return `str:${key}::${idx}:`;
}

function makeDummyMeasure(entity : string) : Syntax.MeasureEntity {
    const match = /^MEASURE_([A-Za-z]+)_([0-9]+)$/.exec(entity);
    assert(match);
    return { unit: match[1], value: 2.0 + parseInt(match[2]) };
}

function parseDate(form : Date|Syntax.DateEntity) : Date {
    if (form instanceof Date)
        return form;

    const now = new Date;
    let year = form.year;
    if (year === undefined || year < 0)
        year = now.getFullYear();
    let month = form.month;
    if (month === undefined || month < 0)
        month = now.getMonth() + 1;
    let day = form.day;
    if (day === undefined || day < 0)
        day = now.getDate();
    let hour = form.hour;
    if (hour === undefined || hour < 0)
        hour = 0;
    let minute = form.minute;
    if (minute === undefined || minute < 0)
        minute = 0;
    let second = form.second;
    if (second === undefined || second < 0)
        second = 0;
    const millisecond = (second - Math.floor(second))*1000;
    second = Math.floor(second);

    return new Date(year, month-1, day, hour, minute, second, millisecond);
}

function entitiesEqual(type : string, one : AnyEntity, two : AnyEntity) : boolean {
    if (one === two)
        return true;
    if (!one || !two)
        return false;
    if (type.startsWith('GENERIC_ENTITY_')) {
        const eone = one as Syntax.GenericEntity;
        const etwo = two as Syntax.GenericEntity;

        if (!eone.value && !etwo.value)
            return eone.display === etwo.display;
        return (eone.value === etwo.value);
    }

    if (type.startsWith('MEASURE_') ||
        type === 'DURATION') {
        const eone = one as Syntax.MeasureEntity;
        const etwo = two as Syntax.MeasureEntity;
        return eone.value === etwo.value && eone.unit === etwo.unit;
    }

    switch (type) {
    case 'CURRENCY': {
        const eone = one as Syntax.MeasureEntity;
        const etwo = two as Syntax.MeasureEntity;
        return eone.value === etwo.value && eone.unit === etwo.unit;
    }
    case 'TIME': {
        const eone = one as Syntax.TimeEntity;
        const etwo = two as Syntax.TimeEntity;
        return eone.hour === etwo.hour &&
            eone.minute === etwo.minute &&
            (eone.second || 0) === (etwo.second || 0);
    }
    case 'DATE':
        if (!(one instanceof Date))
            one = parseDate(one as Syntax.DateEntity);
        if (!(two instanceof Date))
            two = parseDate(two as Syntax.DateEntity);

        return +one === +two;
    case 'LOCATION': {
        const eone = one as Syntax.LocationEntity;
        const etwo = two as Syntax.LocationEntity;
        if (isNaN(eone.latitude) && isNaN(etwo.latitude) && isNaN(eone.longitude) && isNaN(etwo.longitude))
            return eone.display === etwo.display;
        return Math.abs(eone.latitude - etwo.latitude) < 0.01 &&
            Math.abs(eone.longitude - etwo.longitude) < 0.01;
    }
    }

    return false;
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_([0-9]+)$/;

function makeDummyEntity(token : string) : AnyEntity {
    const match = ENTITY_MATCH_REGEX.exec(token);
    if (match === null)
        throw new TypeError(`invalid entity ${token}`);
    const [,entityType,entityIndex] = match;
    assert(!Number.isNaN(Number(entityIndex)), token);

    if (entityType.startsWith('MEASURE_'))
        return makeDummyMeasure(token);
    else if (entityType.startsWith('GENERIC_ENTITY_'))
        return { value: namedString(entityType.substring('GENERIC_'.length), Number(entityIndex)), display: null };
    else if (!(entityType in ENTITIES))
        throw new Error(`missing entity ${token}`);
    else
        return ENTITIES[entityType](Number(entityIndex));
}

function makeDummyEntities(preprocessed : string) : EntityMap {
    const entities : EntityMap = {};
    for (const token of preprocessed.split(' ')) {
        if (/^[A-Z]/.test(token))
            entities[token] = makeDummyEntity(token);
    }
    return entities;
}

interface TokenizerResult {
    entities : EntityMap;
    tokens : string[];
}

function renumberEntities(tokenized : TokenizerResult, context : EntityMap) : void {
    const { entities, tokens } = tokenized;
    const out : EntityMap = {};

    const offsets : { [key : string] : number } = {};
    for (const key in context) {
        const [, type, num] = /^(.+)_([0-9]+)$/.exec(key)!;

        const next = offsets[type] || 0;
        offsets[type] = Math.max(next, parseInt(num, 10) + 1);
        out[key] = context[key];
    }

    const rewrites : { [key : string] : string } = {};

    for (let i = 0; i < tokens.length; i++) {
        if (tokens[i] in rewrites) {
            tokens[i] = rewrites[tokens[i]];
            continue;
        }

        if (tokens[i] in entities) {
            const key = tokens[i];
            const [, type, ] = /^(.+)_([0-9]+)$/.exec(key)!;
            const value = entities[key];

            let found = false;
            for (const what in context) {
                if (!what.startsWith(type + '_'))
                    continue;

                if (entitiesEqual(type, context[what], value)) {
                    found = true;
                    rewrites[key] = what;
                    tokens[i] = what;
                    break;
                }
            }

            if (!found) {
                const next = offsets[type] || 0;
                offsets[type] = next + 1;
                const newkey = type + '_' + next;
                out[newkey] = entities[key];
                rewrites[key] = newkey;
                tokens[i] = newkey;
            }
        }
    }

    tokenized.entities = out;
}

export {
    makeDummyEntity,
    makeDummyEntities,
    renumberEntities,
};

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import assert from 'assert';	61✔
22	import { Syntax } from 'thingtalk';
23
24	export type AnyEntity = Syntax.AnyEntity;
25	export type EntityMap = Syntax.EntityMap;
26
27	const MAX_SMALL_INTEGER = 12;	61✔
28
29	const ENTITIES : { [key : string] : (idx : number) => AnyEntity } = {	61✔
30	DURATION: (idx) => ({ value: idx + 2, unit: 'ms' }),	601✔
31	NUMBER: (idx) => idx + MAX_SMALL_INTEGER + 1,	3,043✔
32	DATE: (idx) => ({ day: idx + 2, month: 1, year: 2018 }),	2,353✔
33	TIME: (idx) => ({ hour: Math.floor(idx/4), minute: [0, 15, 30, 45][idx % 4], second: 0 }),	26,008✔
34	CURRENCY: (idx) => ({ value: idx + 2, unit: 'usd' }),	618✔
35	LOCATION: (idx) => ({ latitude: idx + 2, longitude: idx + 2 }),	288✔
36	QUOTED_STRING: (idx) => namedString('QUOTED_STRING', idx),	192,126✔
37	PATH_NAME: (idx) => namedString('PATH_NAME', idx),	66✔
UNCOV 38	URL: (idx) => namedString('URL', idx),	×
39	PHONE_NUMBER: (idx) => namedString('PHONE_NUMBER', idx),	683✔
UNCOV 40	EMAIL_ADDRESS: (idx) => namedString('EMAIL_ADDRESS', idx),	×
41	USERNAME: (idx) => namedString('USERNAME', idx),	55✔
42	HASHTAG: (idx) => namedString('HASHTAG', idx),	657✔
43	PICTURE: (idx) => namedString('PICTURE', idx)	×
44	};
45	Object.freeze(ENTITIES);	61✔
46
47	function namedString(key : string, idx : number) : string {
48	return `str:${key}::${idx}:`;	193,732✔
49	}
50
51	function makeDummyMeasure(entity : string) : Syntax.MeasureEntity {
52	const match = /^MEASURE_([A-Za-z]+)_([0-9]+)$/.exec(entity);	×
53	assert(match);	×
54	return { unit: match[1], value: 2.0 + parseInt(match[2]) };	×
55	}
56
57	function parseDate(form : Date\|Syntax.DateEntity) : Date {
58	if (form instanceof Date)	×
59	return form;	×
60
61	const now = new Date;	×
62	let year = form.year;	×
63	if (year === undefined \|\| year < 0)	×
64	year = now.getFullYear();	×
65	let month = form.month;	×
66	if (month === undefined \|\| month < 0)	×
67	month = now.getMonth() + 1;	×
68	let day = form.day;	×
69	if (day === undefined \|\| day < 0)	×
70	day = now.getDate();	×
71	let hour = form.hour;	×
72	if (hour === undefined \|\| hour < 0)	×
73	hour = 0;	×
74	let minute = form.minute;	×
75	if (minute === undefined \|\| minute < 0)	×
76	minute = 0;	×
77	let second = form.second;	×
78	if (second === undefined \|\| second < 0)	×
79	second = 0;	×
80	const millisecond = (second - Math.floor(second))*1000;	×
81	second = Math.floor(second);	×
82
83	return new Date(year, month-1, day, hour, minute, second, millisecond);	×
84	}
85
86	function entitiesEqual(type : string, one : AnyEntity, two : AnyEntity) : boolean {
87	if (one === two)	42,871✔
88	return true;	7,703✔
89	if (!one \|\| !two)	35,168!
90	return false;	×
91	if (type.startsWith('GENERIC_ENTITY_')) {	35,168!
92	const eone = one as Syntax.GenericEntity;	×
93	const etwo = two as Syntax.GenericEntity;	×
94
95	if (!eone.value && !etwo.value)	×
96	return eone.display === etwo.display;	×
97	return (eone.value === etwo.value);	×
98	}
99
100	if (type.startsWith('MEASURE_') \|\|	35,168!
101	type === 'DURATION') {
102	const eone = one as Syntax.MeasureEntity;	×
103	const etwo = two as Syntax.MeasureEntity;	×
104	return eone.value === etwo.value && eone.unit === etwo.unit;	×
105	}
106
107	switch (type) {	35,168✔
108	case 'CURRENCY': {	5,491!
109	const eone = one as Syntax.MeasureEntity;	×
110	const etwo = two as Syntax.MeasureEntity;	×
111	return eone.value === etwo.value && eone.unit === etwo.unit;	×
112	}
113	case 'TIME': {
114	const eone = one as Syntax.TimeEntity;	5,491✔
115	const etwo = two as Syntax.TimeEntity;	5,491✔
116	return eone.hour === etwo.hour &&	5,491✔
117	eone.minute === etwo.minute &&
118	(eone.second \|\| 0) === (etwo.second \|\| 0);	4,580✔
119	}
120	case 'DATE':
121	if (!(one instanceof Date))	×
122	one = parseDate(one as Syntax.DateEntity);	×
123	if (!(two instanceof Date))	×
124	two = parseDate(two as Syntax.DateEntity);	×
125
126	return +one === +two;	×
127	case 'LOCATION': {
128	const eone = one as Syntax.LocationEntity;	×
129	const etwo = two as Syntax.LocationEntity;	×
130	if (isNaN(eone.latitude) && isNaN(etwo.latitude) && isNaN(eone.longitude) && isNaN(etwo.longitude))	×
131	return eone.display === etwo.display;	×
132	return Math.abs(eone.latitude - etwo.latitude) < 0.01 &&	×
133	Math.abs(eone.longitude - etwo.longitude) < 0.01;
134	}
135	}
136
137	return false;	29,677✔
138	}
139
140	const ENTITY_MATCH_REGEX = /^([A-Z].*)_([0-9]+)$/;	61✔
141
142	function makeDummyEntity(token : string) : AnyEntity {
143	const match = ENTITY_MATCH_REGEX.exec(token);	226,643✔
144	if (match === null)	226,643!
145	throw new TypeError(`invalid entity ${token}`);	×
146	const [,entityType,entityIndex] = match;	226,643✔
147	assert(!Number.isNaN(Number(entityIndex)), token);	226,643✔
148
149	if (entityType.startsWith('MEASURE_'))	226,643!
150	return makeDummyMeasure(token);	×
151	else if (entityType.startsWith('GENERIC_ENTITY_'))	226,643✔
152	return { value: namedString(entityType.substring('GENERIC_'.length), Number(entityIndex)), display: null };	145✔
153	else if (!(entityType in ENTITIES))	226,498!
154	throw new Error(`missing entity ${token}`);	×
155	else
156	return ENTITIES[entityType](Number(entityIndex));	226,498✔
157	}
158
159	function makeDummyEntities(preprocessed : string) : EntityMap {
160	const entities : EntityMap = {};	649,505✔
161	for (const token of preprocessed.split(' ')) {	649,505✔
162	if (/^[A-Z]/.test(token))	4,603,314✔
163	entities[token] = makeDummyEntity(token);	208,892✔
164	}
165	return entities;	649,505✔
166	}
167
168	interface TokenizerResult {
169	entities : EntityMap;
170	tokens : string[];
171	}
172
173	function renumberEntities(tokenized : TokenizerResult, context : EntityMap) : void {
174	const { entities, tokens } = tokenized;	638,810✔
175	const out : EntityMap = {};	638,810✔
176
177	const offsets : { [key : string] : number } = {};	638,810✔
178	for (const key in context) {	638,810✔
179	const [, type, num] = /^(.+)_([0-9]+)$/.exec(key)!;	85,838✔
180
181	const next = offsets[type] \|\| 0;	85,838✔
182	offsets[type] = Math.max(next, parseInt(num, 10) + 1);	85,838✔
183	out[key] = context[key];	85,838✔
184	}
185
186	const rewrites : { [key : string] : string } = {};	638,810✔
187
188	for (let i = 0; i < tokens.length; i++) {	638,810✔
189	if (tokens[i] in rewrites) {	4,464,980✔
190	tokens[i] = rewrites[tokens[i]];	1,103✔
191	continue;	1,103✔
192	}
193
194	if (tokens[i] in entities) {	4,463,877✔
195	const key = tokens[i];	202,962✔
196	const [, type, ] = /^(.+)_([0-9]+)$/.exec(key)!;	202,962✔
197	const value = entities[key];	202,962✔
198
199	let found = false;	202,962✔
200	for (const what in context) {	202,962✔
201	if (!what.startsWith(type + '_'))	90,810✔
202	continue;	47,939✔
203
204	if (entitiesEqual(type, context[what], value)) {	42,871✔
205	found = true;	8,848✔
206	rewrites[key] = what;	8,848✔
207	tokens[i] = what;	8,848✔
208	break;	8,848✔
209	}
210	}
211
212	if (!found) {	202,962✔
213	const next = offsets[type] \|\| 0;	194,114✔
214	offsets[type] = next + 1;	194,114✔
215	const newkey = type + '_' + next;	194,114✔
216	out[newkey] = entities[key];	194,114✔
217	rewrites[key] = newkey;	194,114✔
218	tokens[i] = newkey;	194,114✔
219	}
220	}
221	}
222
223	tokenized.entities = out;	638,810✔
224	}
225
226	export {
227	makeDummyEntity,	61✔
228	makeDummyEntities,	61✔
229	renumberEntities,	61✔
230	};

stanford-oval / genie-toolkit / 1493

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous