1536

pending completion

Build # 1536

Build Type

push

travis-ci-com

Committed by

gcampax

Commit Message

v2.1.0-alpha.7

Run Details

4615 of 6460 branches covered (71.44%)

Branch coverage included in aggregate %.

1 of 1 new or added line in 1 file covered. (100.0%)

406 existing lines in 14 files now uncovered.

10501 of 12981 relevant lines covered (80.9%)

48308.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.12

/lib/entity-retriever.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of ThingTalk
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import * as util from 'util';

import { parseDate } from './utils/date_utils';
import List from './utils/list';

import { SyntaxType } from './syntax_api';
import {
    MeasureEntity,
    LocationEntity,
    TimeEntity,
    DateEntity,
    GenericEntity,
    AnyEntity,
    EntityMap
} from './entities';
import { Temporal } from '@js-temporal/polyfill';

const EPSILON = 1e-8;

function entitiesEqual(type : string, one : AnyEntity, two : AnyEntity, timezone : string) : boolean {
    if (one === two)
        return true;
    if (!one || !two)
        return false;
    if (type.startsWith('GENERIC_ENTITY_')) {
        const eone = one as GenericEntity;
        const etwo = two as GenericEntity;

        if (!eone.value && !etwo.value)
            return eone.display === etwo.display;
        return (eone.value === etwo.value);
    }

    if (type.startsWith('MEASURE_') ||
        type === 'DURATION') {
        const eone = one as MeasureEntity;
        const etwo = two as MeasureEntity;
        return eone.value === etwo.value && eone.unit === etwo.unit;
    }

    switch (type) {
    case 'CURRENCY': {
        const eone = one as MeasureEntity;
        const etwo = two as MeasureEntity;
        return eone.value === etwo.value && eone.unit === etwo.unit;
    }
    case 'TIME': {
        const eone = one as TimeEntity;
        const etwo = two as TimeEntity;
        return eone.hour === etwo.hour &&
            eone.minute === etwo.minute &&
            (eone.second || 0) === (etwo.second || 0);
    }
    case 'DATE':
        if (!(one instanceof Date))
            one = parseDate(one as DateEntity, timezone);
        if (!(two instanceof Date))
            two = parseDate(two as DateEntity, timezone);

        return +one === +two;
    case 'LOCATION': {
        const eone = one as LocationEntity;
        const etwo = two as LocationEntity;
        if (isNaN(eone.latitude) && isNaN(etwo.latitude) && isNaN(eone.longitude) && isNaN(etwo.longitude))
            return eone.display === etwo.display;
        return Math.abs(eone.latitude - etwo.latitude) < EPSILON &&
            Math.abs(eone.longitude - etwo.longitude) < EPSILON;
    }
    }

    return false;
}

function entityToString(entityType : string, entity : AnyEntity) : string {
    if ((entityType.startsWith('GENERIC_ENTITY_') || entityType === 'LOCATION')) {
        const generic = entity as GenericEntity;
        if (generic.display)
            return generic.display;
        if (generic.value)
            return generic.value;
    }
    return String(entity);
}

/**
 * Abstract class capable of allocating entity numbers when converting
 * ThingTalk code to NN syntax (which uses numbered entities matching the input sentence).
 */
export abstract class AbstractEntityRetriever {
    protected _syntaxType : SyntaxType.Tokenized|SyntaxType.LegacyNN;
    protected _timezone : string;

    constructor(options : {
        timezone : string|undefined
    }) {
        this._timezone = options.timezone ?? Temporal.Now.timeZone().id;
        this._syntaxType = SyntaxType.LegacyNN;
    }

    get timezone() {
        return this._timezone;
    }

    setSyntaxType(syntaxType : SyntaxType.Tokenized|SyntaxType.LegacyNN) {
        this._syntaxType = syntaxType;
    }

    /**
     * Find the entity with the given `entityType` (USERNAME, HASHTAG, etc.) and value.
     *
     * @param entityType - the type of entity to retrieve
     * @param value - the value to retrieve
     * @param options - additional options
     * @param options.ignoreNotFound - return `null` if the entity is not found, instead
     *   of throwing an exception.
     * @return the list of tokens making up this entity.
     */
    abstract findEntity(entityType : string, value : AnyEntity, options : { ignoreNotFound : true, includeEntityValue ?: boolean }) : List<string>|null;
    abstract findEntity(entityType : string, value : AnyEntity, options ?: { ignoreNotFound ?: false, includeEntityValue ?: boolean }) : List<string>;
}

/**
 * Entity retriever that looks for an entity in the tokenized entities, if any, and then
 * falls back to string matching in the sentence.
 */
export class EntityRetriever extends AbstractEntityRetriever {
    sentence : string[];
    entities : EntityMap;

    constructor(sentence : string|string[], entities : EntityMap, options : {
        timezone : string|undefined
    }) {
        super(options);
        if (typeof sentence === 'string')
            sentence = sentence.split(' ');
        this.sentence = sentence;

        this.entities = {};
        Object.assign(this.entities, entities);
    }

    protected _sentenceContains(tokens : string[]) : boolean {
        for (let i = 0; i <= this.sentence.length-tokens.length; i++) {
            let found = true;
            for (let j = 0; j < tokens.length; j++) {
                if (tokens[j] !== this.sentence[i+j]) {
                    found = false;
                    break;
                }
            }
            if (found)
                return true;
        }
        return false;
    }

    /**
     * Match an entity from the sentence.
     *
     * This method should search for the entity string in the sentence, and return the value
     * to predict in NN-syntax, or `undefined` if the entity is not mentioned.
     * This method can be overridden to implement custom tokenization or normalization.
     *
     * @param {string} entityType - the entity type (USERNAME, HASHTAG, QUOTED_STRING, etc.)
     * @param {string} entityString - the string to search
     * @param {boolean} ignoreNotFound - ignore if the entity is not mentioned; subclasses can
     *   use this to hallucinate entities that are not mentioned, when `ignoreNotFound` is false
     * @return the tokens to predict, or `undefined` if the entity is not mentioned in the sentence.
     */
    protected _findEntityFromSentence(entityType : string, entityString : string, ignoreNotFound : boolean) : string[]|undefined {
        const entityTokens = entityString.toLowerCase().split(' ');
        const found = this._sentenceContains(entityTokens);
        if (found)
            return entityTokens;
        else
            return undefined;
    }

    /**
     * Match a numeric entity from the sentence.
     *
     * This method should search for a mention of the number in the sentence, and return the value
     * to predict in NN-syntax, or `undefined` if the entity is not mentioned.
     * This method can be overridden to implement custom tokenization or normalization.
     *
     * @param {string} entityType - the numeric entity type (NUMBER, MEASURE, CURRENCY, etc.)
     * @param {number} number - the number to search
     * @param {boolean} ignoreNotFound - ignore if the number is not mentioned; subclasses can
     *   use this to hallucinate entities that are not mentioned, when `ignoreNotFound` is false
     * @return the tokens to predict, or `undefined` if the entity is not mentioned in the sentence.
     */
    protected _findNumberFromSentence(entityType : string, number : number, ignoreNotFound : boolean) : string[]|undefined {
        // by default, we normalize using JS syntax for numbers: "." for decimal
        // separator, and no thousand separator
        const entityTokens = [String(number)];
        const found = this._sentenceContains(entityTokens);
        if (found)
            return entityTokens;
        else
            return undefined;
    }

    private  _findStringLikeEntity(entityType : string,
                                   entity : AnyEntity,
                                   entityString : string,
                                   ignoreNotFound : boolean,
                                   includeEntityValue : boolean) : List<string>|undefined {
        if (entityType === 'DATE') {
            const dateStr = (entity as Date).toISOString();
            if (this._sentenceContains([dateStr]))
                return List.concat('new', 'Date', '(', '"', dateStr, '"', ')');
        }
        if (entityType === 'NUMBER') {
            const found = this._findNumberFromSentence(entityType, entity as number, ignoreNotFound);
            if (found) // ignore the returned tokens, and always predict normalized English-like syntax
                return List.singleton(String(entity));
        }
        if (entityType === 'CURRENCY' || entityType === 'DURATION' || entityType.startsWith('MEASURE_')) {
            const measure = entity as MeasureEntity;
            const found = this._findNumberFromSentence(entityType, measure.value, ignoreNotFound);
            if (found) // ignore the returned tokens, and always predict normalized English-like syntax
                return List.concat(String(measure.value), entityType === 'CURRENCY' ? ('$' + measure.unit) : measure.unit);
        }

        if (entityType === 'QUOTED_STRING' || entityType === 'HASHTAG' || entityType === 'USERNAME' ||
            entityType === 'PATH_NAME' || entityType === 'URL' || entityType === 'PHONE_NUMBER' ||
            entityType === 'EMAIL_ADDRESS' || entityType === 'LOCATION' || entityType.startsWith('GENERIC_ENTITY_')) {

            const found = this._findEntityFromSentence(entityType, entityString, ignoreNotFound);
            if (found) {
                if (entityType === 'QUOTED_STRING')
                    return List.concat('"', ...found, '"');
                else if (entityType === 'HASHTAG')
                    return List.concat('"', ...found, '"', '^^tt:hashtag');
                else if (entityType === 'USERNAME')
                    return List.concat('"', ...found, '"', '^^tt:username');
                else if (entityType === 'PATH_NAME')
                    return List.concat('"', ...found, '"', '^^tt:path_name');
                else if (entityType === 'URL')
                    return List.concat('"', ...found, '"', '^^tt:url');
                else if (entityType === 'PHONE_NUMBER')
                    return List.concat('"', ...found, '"', '^^tt:phone_number');
                else if (entityType === 'EMAIL_ADDRESS')
                    return List.concat('"', ...found, '"', '^^tt:email_address');

                if (this._syntaxType === SyntaxType.LegacyNN) {
                    if (entityType === 'LOCATION')
                        return List.concat('location:', '"', ...found, '"');
                    else
                        return List.concat('"', ...found, '"', '^^' + entityType.substring('GENERIC_ENTITY_'.length));
                } else {
                    if (entityType === 'LOCATION') {
                        return List.concat('new', 'Location', '(', '"', ...found, '"', ')');
                    } else {
                        const genericEntity = entity as GenericEntity;
                        const entityId = includeEntityValue && genericEntity.value ? [ '"', ...genericEntity.value.split(' '), '"'] : ['null'];
                        return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', '"', ...found, '"', ')');
                    }
                }
            }
        }

        // always predict (not copy) these entities if they are missing from the sentence
        // (the neural model will learn the names of the devices
        if (entityType === 'GENERIC_ENTITY_tt:device') {
            const value = (entity as GenericEntity).value!;
            if (this._syntaxType === SyntaxType.LegacyNN)
                return List.singleton('device:' + value);
            else
                return List.singleton('@' + value);
        }
        if (entityType === 'GENERIC_ENTITY_tt:function') {
            const value = (entity as GenericEntity).value!;
            if (this._syntaxType === SyntaxType.LegacyNN) {
                return List.singleton('@' + value);
            } else {
                const dot = value.lastIndexOf('.');
                const kind = value.substring(0, dot);
                const name = value.substring(dot+1, value.length);
                return List.concat('@' + kind, '.', name);
            }
        }

        return undefined;
    }

    private _findEntityInBag(entityType : string, value : AnyEntity, entities : EntityMap) : List<string>|undefined {
        for (const what in entities) {
            if (!what.startsWith(entityType + '_'))
                continue;

            if (entitiesEqual(entityType, entities[what], value, this._timezone))
                return List.singleton(what);
        }
        return undefined;
    }

    findEntity(entityType : string, entity : AnyEntity, options : { ignoreNotFound : true, includeEntityValue ?: boolean }) : List<string>|null;
    findEntity(entityType : string, entity : AnyEntity, options ?: { ignoreNotFound ?: false, includeEntityValue ?: boolean }) : List<string>;
    findEntity(entityType : string, entity : AnyEntity, { ignoreNotFound = false, includeEntityValue = false } = {}) : List<string>|null {
        const entityString = entityToString(entityType, entity);

        // try in the sentence before we look in the bag of entities (which comes from the context)
        // this is so that we predict
        // " foo " ^^tt:whatever
        // if the sentence contains "foo", regardless of whether GENERIC_ENTITY_tt:whatever_0 is "foo" or not
        let found = this._findStringLikeEntity(entityType, entity, entityString, true, includeEntityValue);
        if (found)
            return found;

        found = this._findEntityInBag(entityType, entity, this.entities);
        if (found)
            return found;

        if (ignoreNotFound)
            return null;

        if (entityType.startsWith('GENERIC_ENTITY_') && this._syntaxType === SyntaxType.Tokenized) {
            const genericEntity = entity as GenericEntity;
            if (genericEntity.display) {
                found = this._findEntityInBag('QUOTED_STRING', genericEntity.display, this.entities);
                if (found) {
                    const entityId = includeEntityValue && genericEntity.value ? ['"', genericEntity.value, '"'] : ['null'];
                    return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', found, ')');
                }
            }
        }

        found = this._findStringLikeEntity(entityType, entity, entityString, false, includeEntityValue);
        if (found)
            return found;

        throw new Error(`Cannot find entity ${entityString} of type ${entityType}, have ${util.inspect(this.entities)}`);
    }
}

export class SequentialEntityAllocator extends AbstractEntityRetriever {
    offsets : { [key : string] : number };
    entities : EntityMap;
    explicitStrings : boolean;

    constructor(entities : EntityMap, options : {
        timezone : string|undefined,
        explicitStrings ?: boolean
    }) {
        super(options);
        this.offsets = {};
        this.entities = entities;
        this.explicitStrings = !!options.explicitStrings;
        this.updateOffsets();
    }

    reset() {
        this.offsets = {};
        this.entities = {};
    }

    private updateOffsets() : void {
        for (const entity in this.entities) {
            const entityType = entity.slice(0, entity.lastIndexOf('_'));
            const offset = entity.slice(entity.lastIndexOf('_') + 1);
            assert(/^\d+$/.test(offset));
            this.offsets[entityType] = Math.max((this.offsets[entityType] || -1), parseInt(offset) + 1);
        }
    }

    findEntity(entityType : string, entity : AnyEntity, { ignoreNotFound = false, includeEntityValue = false } = {}) : List<string> {
        if (this.explicitStrings &&
            (entityType === 'QUOTED_STRING' || entityType === 'HASHTAG' || entityType === 'USERNAME' ||
            entityType === 'LOCATION' || entityType.startsWith('GENERIC_ENTITY_'))) {
            const entityString = entityToString(entityType, entity).split(' ');

            if (entityType === 'QUOTED_STRING')
                return List.concat('"', ...entityString, '"');
            else if (entityType === 'HASHTAG')
                return List.concat('"', ...entityString, '"', '^^tt:hashtag');
            else if (entityType === 'USERNAME')
                return List.concat('"', ...entityString, '"', '^^tt:username');

            if (this._syntaxType === SyntaxType.LegacyNN) {
                if (entityType === 'LOCATION')
                    return List.concat('location:', '"', ...entityString, '"');
                else
                    return List.concat('"', ...entityString, '"', '^^' + entityType.substring('GENERIC_ENTITY_'.length));
            } else {
                if (entityType === 'LOCATION') {
                    return List.concat('new', 'Location', '(', '"', ...entityString, '"', ')');
                } else {
                    const genericEntity = entity as GenericEntity;
                    const entityId = includeEntityValue && genericEntity.value ? ['"', genericEntity.value, '"'] : ['null'];
                    return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', '"', ...entityString, '"', ')');
                }
            }
        }

        for (const what in this.entities) {
            if (!what.startsWith(entityType + '_'))
                continue;

            if (entitiesEqual(entityType, this.entities[what], entity, this._timezone))
                return List.singleton(what);
        }

        let num;
        if (entityType in this.offsets) {
            num = this.offsets[entityType];
            this.offsets[entityType] += 1;
        } else {
            num = 0;
            this.offsets[entityType] = 1;
        }

        const key = entityType + '_' + num;
        this.entities[key] = entity;
        return List.singleton(key);
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of ThingTalk
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import assert from 'assert';	2✔
22	import * as util from 'util';	2✔
23
24	import { parseDate } from './utils/date_utils';	2✔
25	import List from './utils/list';	2✔
26
27	import { SyntaxType } from './syntax_api';	2✔
28	import {
29	MeasureEntity,
30	LocationEntity,
31	TimeEntity,
32	DateEntity,
33	GenericEntity,
34	AnyEntity,
35	EntityMap
36	} from './entities';
37	import { Temporal } from '@js-temporal/polyfill';	2✔
38
39	const EPSILON = 1e-8;	2✔
40
41	function entitiesEqual(type : string, one : AnyEntity, two : AnyEntity, timezone : string) : boolean {
42	if (one === two)	790✔
43	return true;	486✔
44	if (!one \|\| !two)	304!
UNCOV 45	return false;	×
46	if (type.startsWith('GENERIC_ENTITY_')) {	304✔
47	const eone = one as GenericEntity;	80✔
48	const etwo = two as GenericEntity;	80✔
49
50	if (!eone.value && !etwo.value)	80✔
51	return eone.display === etwo.display;	4✔
52	return (eone.value === etwo.value);	76✔
53	}
54
55	if (type.startsWith('MEASURE_') \|\|	224✔
56	type === 'DURATION') {
57	const eone = one as MeasureEntity;	26✔
58	const etwo = two as MeasureEntity;	26✔
59	return eone.value === etwo.value && eone.unit === etwo.unit;	26✔
60	}
61
62	switch (type) {	198✔
63	case 'CURRENCY': {	88✔
64	const eone = one as MeasureEntity;	6✔
65	const etwo = two as MeasureEntity;	6✔
66	return eone.value === etwo.value && eone.unit === etwo.unit;	6✔
67	}
68	case 'TIME': {
69	const eone = one as TimeEntity;	40✔
70	const etwo = two as TimeEntity;	40✔
71	return eone.hour === etwo.hour &&	40✔
72	eone.minute === etwo.minute &&
73	(eone.second \|\| 0) === (etwo.second \|\| 0);	136✔
74	}
75	case 'DATE':
76	if (!(one instanceof Date))	30!
77	one = parseDate(one as DateEntity, timezone);	30✔
78	if (!(two instanceof Date))	30!
UNCOV 79	two = parseDate(two as DateEntity, timezone);	×
80
81	return +one === +two;	30✔
82	case 'LOCATION': {
83	const eone = one as LocationEntity;	12✔
84	const etwo = two as LocationEntity;	12✔
85	if (isNaN(eone.latitude) && isNaN(etwo.latitude) && isNaN(eone.longitude) && isNaN(etwo.longitude))	12✔
86	return eone.display === etwo.display;	4✔
87	return Math.abs(eone.latitude - etwo.latitude) < EPSILON &&	8✔
88	Math.abs(eone.longitude - etwo.longitude) < EPSILON;
89	}
90	}
91
92	return false;	110✔
93	}
94
95	function entityToString(entityType : string, entity : AnyEntity) : string {
96	if ((entityType.startsWith('GENERIC_ENTITY_') \|\| entityType === 'LOCATION')) {	826✔
97	const generic = entity as GenericEntity;	98✔
98	if (generic.display)	98✔
99	return generic.display;	36✔
100	if (generic.value)	62!
101	return generic.value;	62✔
102	}
103	return String(entity);	728✔
104	}
105
106	/**
107	* Abstract class capable of allocating entity numbers when converting
108	* ThingTalk code to NN syntax (which uses numbered entities matching the input sentence).
109	*/
110	export abstract class AbstractEntityRetriever {	2✔
111	protected _syntaxType : SyntaxType.Tokenized\|SyntaxType.LegacyNN;
112	protected _timezone : string;
113
114	constructor(options : {
115	timezone : string\|undefined
116	}) {
117	this._timezone = options.timezone ?? Temporal.Now.timeZone().id;	760!
118	this._syntaxType = SyntaxType.LegacyNN;	760✔
119	}
120
121	get timezone() {
122	return this._timezone;	8✔
123	}
124
125	setSyntaxType(syntaxType : SyntaxType.Tokenized\|SyntaxType.LegacyNN) {
126	this._syntaxType = syntaxType;	760✔
127	}
128
129	/**
130	* Find the entity with the given `entityType` (USERNAME, HASHTAG, etc.) and value.
131	*
132	* @param entityType - the type of entity to retrieve
133	* @param value - the value to retrieve
134	* @param options - additional options
135	* @param options.ignoreNotFound - return `null` if the entity is not found, instead
136	* of throwing an exception.
137	* @return the list of tokens making up this entity.
138	*/
139	abstract findEntity(entityType : string, value : AnyEntity, options : { ignoreNotFound : true, includeEntityValue ?: boolean }) : List<string>\|null;
140	abstract findEntity(entityType : string, value : AnyEntity, options ?: { ignoreNotFound ?: false, includeEntityValue ?: boolean }) : List<string>;
141	}
142
143	/**
144	* Entity retriever that looks for an entity in the tokenized entities, if any, and then
145	* falls back to string matching in the sentence.
146	*/
147	export class EntityRetriever extends AbstractEntityRetriever {	2✔
148	sentence : string[];
149	entities : EntityMap;
150
151	constructor(sentence : string\|string[], entities : EntityMap, options : {
152	timezone : string\|undefined
153	}) {
154	super(options);	660✔
155	if (typeof sentence === 'string')	660!
156	sentence = sentence.split(' ');	660✔
157	this.sentence = sentence;	660✔
158
159	this.entities = {};	660✔
160	Object.assign(this.entities, entities);	660✔
161	}
162
163	protected _sentenceContains(tokens : string[]) : boolean {
164	for (let i = 0; i <= this.sentence.length-tokens.length; i++) {	688✔
165	let found = true;	5,670✔
166	for (let j = 0; j < tokens.length; j++) {	5,670✔
167	if (tokens[j] !== this.sentence[i+j]) {	5,688✔
168	found = false;	5,596✔
169	break;	5,596✔
170	}
171	}
172	if (found)	5,670✔
173	return true;	74✔
174	}
175	return false;	614✔
176	}
177
178	/**
179	* Match an entity from the sentence.
180	*
181	* This method should search for the entity string in the sentence, and return the value
182	* to predict in NN-syntax, or `undefined` if the entity is not mentioned.
183	* This method can be overridden to implement custom tokenization or normalization.
184	*
185	* @param {string} entityType - the entity type (USERNAME, HASHTAG, QUOTED_STRING, etc.)
186	* @param {string} entityString - the string to search
187	* @param {boolean} ignoreNotFound - ignore if the entity is not mentioned; subclasses can
188	* use this to hallucinate entities that are not mentioned, when `ignoreNotFound` is false
189	* @return the tokens to predict, or `undefined` if the entity is not mentioned in the sentence.
190	*/
191	protected _findEntityFromSentence(entityType : string, entityString : string, ignoreNotFound : boolean) : string[]\|undefined {
192	const entityTokens = entityString.toLowerCase().split(' ');	352✔
193	const found = this._sentenceContains(entityTokens);	352✔
194	if (found)	352✔
195	return entityTokens;	60✔
196	else
197	return undefined;	292✔
198	}
199
200	/**
201	* Match a numeric entity from the sentence.
202	*
203	* This method should search for a mention of the number in the sentence, and return the value
204	* to predict in NN-syntax, or `undefined` if the entity is not mentioned.
205	* This method can be overridden to implement custom tokenization or normalization.
206	*
207	* @param {string} entityType - the numeric entity type (NUMBER, MEASURE, CURRENCY, etc.)
208	* @param {number} number - the number to search
209	* @param {boolean} ignoreNotFound - ignore if the number is not mentioned; subclasses can
210	* use this to hallucinate entities that are not mentioned, when `ignoreNotFound` is false
211	* @return the tokens to predict, or `undefined` if the entity is not mentioned in the sentence.
212	*/
213	protected _findNumberFromSentence(entityType : string, number : number, ignoreNotFound : boolean) : string[]\|undefined {
214	// by default, we normalize using JS syntax for numbers: "." for decimal
215	// separator, and no thousand separator
216	const entityTokens = [String(number)];	284✔
217	const found = this._sentenceContains(entityTokens);	284✔
218	if (found)	284✔
219	return entityTokens;	8✔
220	else
221	return undefined;	276✔
222	}
223
224	private _findStringLikeEntity(entityType : string,
225	entity : AnyEntity,
226	entityString : string,
227	ignoreNotFound : boolean,
228	includeEntityValue : boolean) : List<string>\|undefined {
229	if (entityType === 'DATE') {	826✔
230	const dateStr = (entity as Date).toISOString();	52✔
231	if (this._sentenceContains([dateStr]))	52✔
232	return List.concat('new', 'Date', '(', '"', dateStr, '"', ')');	6✔
233	}
234	if (entityType === 'NUMBER') {	820✔
235	const found = this._findNumberFromSentence(entityType, entity as number, ignoreNotFound);	222✔
236	if (found) // ignore the returned tokens, and always predict normalized English-like syntax	222✔
237	return List.singleton(String(entity));	4✔
238	}
239	if (entityType === 'CURRENCY' \|\| entityType === 'DURATION' \|\| entityType.startsWith('MEASURE_')) {	816✔
240	const measure = entity as MeasureEntity;	62✔
241	const found = this._findNumberFromSentence(entityType, measure.value, ignoreNotFound);	62✔
242	if (found) // ignore the returned tokens, and always predict normalized English-like syntax	62✔
243	return List.concat(String(measure.value), entityType === 'CURRENCY' ? ('$' + measure.unit) : measure.unit);	4✔
244	}
245
246	if (entityType === 'QUOTED_STRING' \|\| entityType === 'HASHTAG' \|\| entityType === 'USERNAME' \|\|	812✔
247	entityType === 'PATH_NAME' \|\| entityType === 'URL' \|\| entityType === 'PHONE_NUMBER' \|\|
248	entityType === 'EMAIL_ADDRESS' \|\| entityType === 'LOCATION' \|\| entityType.startsWith('GENERIC_ENTITY_')) {
249
250	const found = this._findEntityFromSentence(entityType, entityString, ignoreNotFound);	352✔
251	if (found) {	352✔
252	if (entityType === 'QUOTED_STRING')	60✔
253	return List.concat('"', ...found, '"');	22✔
254	else if (entityType === 'HASHTAG')	38✔
255	return List.concat('"', ...found, '"', '^^tt:hashtag');	12✔
256	else if (entityType === 'USERNAME')	26✔
257	return List.concat('"', ...found, '"', '^^tt:username');	12✔
258	else if (entityType === 'PATH_NAME')	14!
UNCOV 259	return List.concat('"', ...found, '"', '^^tt:path_name');	×
260	else if (entityType === 'URL')	14!
UNCOV 261	return List.concat('"', ...found, '"', '^^tt:url');	×
262	else if (entityType === 'PHONE_NUMBER')	14!
UNCOV 263	return List.concat('"', ...found, '"', '^^tt:phone_number');	×
264	else if (entityType === 'EMAIL_ADDRESS')	14!
UNCOV 265	return List.concat('"', ...found, '"', '^^tt:email_address');	×
266
267	if (this._syntaxType === SyntaxType.LegacyNN) {	14✔
268	if (entityType === 'LOCATION')	8✔
269	return List.concat('location:', '"', ...found, '"');	4✔
270	else
271	return List.concat('"', ...found, '"', '^^' + entityType.substring('GENERIC_ENTITY_'.length));	4✔
272	} else {
273	if (entityType === 'LOCATION') {	6✔
274	return List.concat('new', 'Location', '(', '"', ...found, '"', ')');	2✔
275	} else {
276	const genericEntity = entity as GenericEntity;	4✔
277	const entityId = includeEntityValue && genericEntity.value ? [ '"', ...genericEntity.value.split(' '), '"'] : ['null'];	4!
278	return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', '"', ...found, '"', ')');	4✔
279	}
280	}
281	}
282	}
283
284	// always predict (not copy) these entities if they are missing from the sentence
285	// (the neural model will learn the names of the devices
286	if (entityType === 'GENERIC_ENTITY_tt:device') {	752✔
287	const value = (entity as GenericEntity).value!;	2✔
288	if (this._syntaxType === SyntaxType.LegacyNN)	2!
289	return List.singleton('device:' + value);	×
290	else
291	return List.singleton('@' + value);	2✔
292	}
293	if (entityType === 'GENERIC_ENTITY_tt:function') {	750!
UNCOV 294	const value = (entity as GenericEntity).value!;	×
UNCOV 295	if (this._syntaxType === SyntaxType.LegacyNN) {	×
UNCOV 296	return List.singleton('@' + value);	×
297	} else {
UNCOV 298	const dot = value.lastIndexOf('.');	×
UNCOV 299	const kind = value.substring(0, dot);	×
UNCOV 300	const name = value.substring(dot+1, value.length);	×
UNCOV 301	return List.concat('@' + kind, '.', name);	×
302	}
303	}
304
305	return undefined;	750✔
306	}
307
308	private _findEntityInBag(entityType : string, value : AnyEntity, entities : EntityMap) : List<string>\|undefined {
309	for (const what in entities) {	754✔
310	if (!what.startsWith(entityType + '_'))	1,150✔
311	continue;	410✔
312
313	if (entitiesEqual(entityType, entities[what], value, this._timezone))	740✔
314	return List.singleton(what);	656✔
315	}
316	return undefined;	98✔
317	}
318
319	findEntity(entityType : string, entity : AnyEntity, options : { ignoreNotFound : true, includeEntityValue ?: boolean }) : List<string>\|null;
320	findEntity(entityType : string, entity : AnyEntity, options ?: { ignoreNotFound ?: false, includeEntityValue ?: boolean }) : List<string>;
321	findEntity(entityType : string, entity : AnyEntity, { ignoreNotFound = false, includeEntityValue = false } = {}) : List<string>\|null {	1,496✔
322	const entityString = entityToString(entityType, entity);	826✔
323
324	// try in the sentence before we look in the bag of entities (which comes from the context)
325	// this is so that we predict
326	// " foo " ^^tt:whatever
327	// if the sentence contains "foo", regardless of whether GENERIC_ENTITY_tt:whatever_0 is "foo" or not
328	let found = this._findStringLikeEntity(entityType, entity, entityString, true, includeEntityValue);	826✔
329	if (found)	826✔
330	return found;	76✔
331
332	found = this._findEntityInBag(entityType, entity, this.entities);	750✔
333	if (found)	750✔
334	return found;	652✔
335
336	if (ignoreNotFound)	98✔
337	return null;	94✔
338
339	if (entityType.startsWith('GENERIC_ENTITY_') && this._syntaxType === SyntaxType.Tokenized) {	4!
340	const genericEntity = entity as GenericEntity;	4✔
341	if (genericEntity.display) {	4!
342	found = this._findEntityInBag('QUOTED_STRING', genericEntity.display, this.entities);	4✔
343	if (found) {	4!
344	const entityId = includeEntityValue && genericEntity.value ? ['"', genericEntity.value, '"'] : ['null'];	4!
345	return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', found, ')');	4✔
346	}
347	}
348	}
349
UNCOV 350	found = this._findStringLikeEntity(entityType, entity, entityString, false, includeEntityValue);	×
UNCOV 351	if (found)	×
UNCOV 352	return found;	×
353
UNCOV 354	throw new Error(`Cannot find entity ${entityString} of type ${entityType}, have ${util.inspect(this.entities)}`);	×
355	}
356	}
357
358	export class SequentialEntityAllocator extends AbstractEntityRetriever {	2✔
359	offsets : { [key : string] : number };
360	entities : EntityMap;
361	explicitStrings : boolean;
362
363	constructor(entities : EntityMap, options : {
364	timezone : string\|undefined,
365	explicitStrings ?: boolean
366	}) {
367	super(options);	100✔
368	this.offsets = {};	100✔
369	this.entities = entities;	100✔
370	this.explicitStrings = !!options.explicitStrings;	100✔
371	this.updateOffsets();	100✔
372	}
373
374	reset() {
UNCOV 375	this.offsets = {};	×
UNCOV 376	this.entities = {};	×
377	}
378
379	private updateOffsets() : void {
380	for (const entity in this.entities) {	100✔
381	const entityType = entity.slice(0, entity.lastIndexOf('_'));	16✔
382	const offset = entity.slice(entity.lastIndexOf('_') + 1);	16✔
383	assert(/^\d+$/.test(offset));	16✔
384	this.offsets[entityType] = Math.max((this.offsets[entityType] \|\| -1), parseInt(offset) + 1);	16✔
385	}
386	}
387
388	findEntity(entityType : string, entity : AnyEntity, { ignoreNotFound = false, includeEntityValue = false } = {}) : List<string> {	212✔
389	if (this.explicitStrings &&	132!
390	(entityType === 'QUOTED_STRING' \|\| entityType === 'HASHTAG' \|\| entityType === 'USERNAME' \|\|
391	entityType === 'LOCATION' \|\| entityType.startsWith('GENERIC_ENTITY_'))) {
UNCOV 392	const entityString = entityToString(entityType, entity).split(' ');	×
393
394	if (entityType === 'QUOTED_STRING')	×
UNCOV 395	return List.concat('"', ...entityString, '"');	×
396	else if (entityType === 'HASHTAG')	×
397	return List.concat('"', ...entityString, '"', '^^tt:hashtag');	×
398	else if (entityType === 'USERNAME')	×
UNCOV 399	return List.concat('"', ...entityString, '"', '^^tt:username');	×
400
UNCOV 401	if (this._syntaxType === SyntaxType.LegacyNN) {	×
UNCOV 402	if (entityType === 'LOCATION')	×
UNCOV 403	return List.concat('location:', '"', ...entityString, '"');	×
404	else
UNCOV 405	return List.concat('"', ...entityString, '"', '^^' + entityType.substring('GENERIC_ENTITY_'.length));	×
406	} else {
UNCOV 407	if (entityType === 'LOCATION') {	×
UNCOV 408	return List.concat('new', 'Location', '(', '"', ...entityString, '"', ')');	×
409	} else {
UNCOV 410	const genericEntity = entity as GenericEntity;	×
UNCOV 411	const entityId = includeEntityValue && genericEntity.value ? ['"', genericEntity.value, '"'] : ['null'];	×
UNCOV 412	return List.concat(...entityId, '^^' + entityType.substring('GENERIC_ENTITY_'.length), '(', '"', ...entityString, '"', ')');	×
413	}
414	}
415	}
416
417	for (const what in this.entities) {	132✔
418	if (!what.startsWith(entityType + '_'))	64✔
419	continue;	14✔
420
421	if (entitiesEqual(entityType, this.entities[what], entity, this._timezone))	50✔
422	return List.singleton(what);	10✔
423	}
424
425	let num;
426	if (entityType in this.offsets) {	122✔
427	num = this.offsets[entityType];	36✔
428	this.offsets[entityType] += 1;	36✔
429	} else {
430	num = 0;	86✔
431	this.offsets[entityType] = 1;	86✔
432	}
433
434	const key = entityType + '_' + num;	122✔
435	this.entities[key] = entity;	122✔
436	return List.singleton(key);	122✔
437	}
438	}

stanford-oval / thingtalk / 1536

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous