2394

pending completion

Build # 2394

Build Type

Pull #806

travis-ci-com

Committed by

web-flow

Commit Message

Merge 3248ddc69 into 7b1a8f0e5

Pull Request Pull Request #806: Wip/translation api

Run Details

10158 of 20582 branches covered (49.35%)

Branch coverage included in aggregate %.

11 of 51 new or added lines in 3 files covered. (21.57%)

3 existing lines in 2 files now uncovered.

24325 of 40776 relevant lines covered (59.66%)

338939.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.53

/lib/prediction/localparserclient.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import * as Tp from 'thingpedia';
import * as ThingTalk from 'thingtalk';

import * as Utils from '../utils/misc-utils';
import * as I18n from '../i18n';
import editDistance from '../utils/edit-distance';
import { EntityMap, renumberEntities } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';

import Predictor from './predictor';
import ExactMatcherBuilder from './exactbuilder';
import {
    ParseOptions,
    PredictionCandidate,
    PredictionResult,
    GenerationResult,
    ExactMatcher
} from './types';

const SEMANTIC_PARSING_TASK = 'almond';
const NLU_TASK = 'almond_dialogue_nlu';
const NLG_TASK = 'almond_dialogue_nlg';
const Translation_TASK = 'almond_translate';
const NLG_QUESTION = 'what should the agent say ?';

export interface LocalParserOptions {
    id ?: string;
    minibatchSize ?: number;
    maxLatency ?: number;
}

function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
    if (a.score === b.score)
        return 0;
    if (a.score === 'Infinity')
        return -1;
    if (b.score === 'Infinity')
        return 1;
    return b.score - a.score;
}

function substringSpan(sequence : string[], substring : string[]) : [number, number] | null {
    for (let i=0; i < sequence.length; i++) {
        let found = true;
        for (let j = 0; j < substring.length; j++) {
            if (sequence[i+j] !== substring[j]) {
                found = false;
                break;
            }
        }
        if (found)
            return [i, i + substring.length + 1];
    }
    return null;
}


export default class LocalParserClient {
    private _locale : string;
    private _langPack : I18n.LanguagePack;
    private _tokenizer : I18n.BaseTokenizer;
    private _predictor : Predictor;
    private _platform : Tp.BasePlatform|undefined;
    private _exactmatcher : ExactMatcher|undefined;
    private _tpClient : Tp.BaseClient|null;

    constructor(modeldir : string,
                locale : string,
                platform : Tp.BasePlatform|undefined,
                exactmatcher : ExactMatcher|undefined,
                tpClient : Tp.BaseClient|null = null,
                options : LocalParserOptions = {}) {
        this._locale = locale;
        this._langPack = I18n.get(locale);
        this._tokenizer = this._langPack.getTokenizer();
        this._predictor = new Predictor(modeldir, options);

        this._platform = platform;
        this._exactmatcher = exactmatcher;
        this._tpClient = tpClient;
    }

    async start() : Promise<void> {
        await Promise.all([
            this._predictor.start(),
            this._startExactMatcher()
        ]);
    }
    async stop() : Promise<void> {
        await this._predictor.stop();
    }

    private async _startExactMatcher() {
        if (this._exactmatcher)
            return;
        if (!this._platform || !this._tpClient)
            return;

        const prefs = this._platform.getSharedPreferences();
        const developerDir = prefs.get('developer-dir') as string|string[];
        if (!developerDir)
            return;
        const builder = new ExactMatcherBuilder({
            locale: this._locale,
            timezone: this._platform.timezone,
            cachedir: this._platform.getCacheDir(),
            developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],
            thingpediaClient: this._tpClient,
        });
        this._exactmatcher = await builder.load();
    }

    private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]|undefined) {
        // only work on contextual
        if (!contextCode)
            return;

        if (contextCode[0] === '$dialogue' &&
            contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
            contextCode[3] === 'sys_anything_else') {
            for (const prog of programs) {
                if (prog.code[0] === '$dialogue' &&
                    prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
                    prog.code[3] === 'cancel')
                    prog.code[3] = 'end';
            }
        }
    }

    async sendUtterance(utterance : string,
                        contextCode : string[]|undefined,
                        contextEntities : EntityMap|undefined,
                        options : ParseOptions = {}) : Promise<PredictionResult> {
        let tokens : string[], entities : EntityMap;
        if (options.tokenized) {
            tokens = utterance.split(' ');
            entities = Utils.makeDummyEntities(utterance);
            if (contextEntities) {
                // safety against weird properties
                for (const key of Object.getOwnPropertyNames(contextEntities)) {
                    if (/^(.+)_([0-9]+)$/.test(key))
                        entities[key] = contextEntities[key];
                }
            }
        } else {
            const tokenized = await this._tokenizer.tokenize(utterance);
            if (contextEntities)
                renumberEntities(tokenized, contextEntities);
            tokens = tokenized.tokens;
            entities = tokenized.entities;
        }

        const answer = options.answer;

        let result : PredictionCandidate[]|null = null;
        let exact : string[][]|null = null;
        const intent = {
            command: 1,
            other: 0,
            ignore: 0
        };

        if (tokens.length === 0) {
            result = [{
                code: ['$failed', ';'],
                score: 'Infinity'
            }];
        } else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) || tokens[0] === '1' || tokens[0] === '0')) {
            // if the whole input is just an entity, return that as an answer
            result = [{
                code: ['$answer', '(', tokens[0], ')', ';'],
                score: 'Infinity'
            }];
        } else if (options.expect === 'MultipleChoice') {
            const choices = await Promise.all((options.choices || []).map((choice) => this._tokenizer.tokenize(choice)));
            result = choices.map((choice, i) => {
                const distance = editDistance(tokens, choice.tokens);
                return {
                    code: ['$choice', '(', String(i), ')', ';'],
                    // distance is between 0 (exact match) and the maximum of the two lengths
                    // adjust distance to be between 0 and 1, higher is better
                    score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
                };
            });
            result.sort(compareScore);
        } else {
            if (this._exactmatcher)
                exact = this._exactmatcher.get(tokens);
        }

        if (result === null) {
            let candidates;
            if (contextCode)
                candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);
            else
                candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);
            assert(candidates.length > 0);

            result = candidates.map((c) => {
                const score = c.score.is_correct ?? 1;
                return {
                    code: c.answer.split(' '),
                    score: score
                };
            });

            intent.other = candidates[0].score.is_ood ?? 0;
            intent.command = 1 - intent.other;
        }

        let result2 = result!; // guaranteed not null
        if (exact !== null)
            result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);

        this._applyPostHeuristics(result2, contextCode);

        if (!options.skip_typechecking) {
            const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);

            result2 = (await Promise.all(result2.map(async (c) => {
                const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {
                    timezone: this._platform?.timezone,
                    thingpediaClient: this._tpClient,
                    schemaRetriever: schemas
                });

                if (parsed) {
                    return {
                        code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
                            locale: this._locale,
                            timezone: this._platform?.timezone,
                            compatibility: options.thingtalk_version,
                            ignoreSentence: true
                        }),
                        score: c.score
                    };
                } else {
                    return null;
                }
            }))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);
        }

        return {
            result: 'ok',
            tokens: tokens,
            candidates: result2,
            entities: entities,
            intent
        };
    }

    async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
        const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }

    async translateUtterance(input : string[], contextEntities : EntityMap|undefined, translationOptions : Record<string, any>) : Promise<GenerationResult[]> {
        if (contextEntities) {
            const allEntities = Object.keys(contextEntities).map((ent) => ent.split(' '));
            for (const entity of allEntities) {
                const span = substringSpan(input, entity);
                if (span) {
                    input.splice(span[0], 0, '"');
                    input.splice(span[1], 0, '"');
                }
            }
        }
        const candidates = await this._predictor.predict('', input.join(' '), undefined, Translation_TASK, 'id-null', translationOptions);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import assert from 'assert';	95✔
22	import * as Tp from 'thingpedia';
23	import * as ThingTalk from 'thingtalk';	95✔
24
25	import * as Utils from '../utils/misc-utils';	95✔
26	import * as I18n from '../i18n';	95✔
27	import editDistance from '../utils/edit-distance';	95✔
28	import { EntityMap, renumberEntities } from '../utils/entity-utils';	95✔
29	import * as ThingTalkUtils from '../utils/thingtalk';	95✔
30
31	import Predictor from './predictor';	95✔
32	import ExactMatcherBuilder from './exactbuilder';	95✔
33	import {
34	ParseOptions,
35	PredictionCandidate,
36	PredictionResult,
37	GenerationResult,
38	ExactMatcher
39	} from './types';
40
41	const SEMANTIC_PARSING_TASK = 'almond';	95✔
42	const NLU_TASK = 'almond_dialogue_nlu';	95✔
43	const NLG_TASK = 'almond_dialogue_nlg';	95✔
44	const Translation_TASK = 'almond_translate';	95✔
45	const NLG_QUESTION = 'what should the agent say ?';	95✔
46
47	export interface LocalParserOptions {
48	id ?: string;
49	minibatchSize ?: number;
50	maxLatency ?: number;
51	}
52
53	function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
54	if (a.score === b.score)	×
55	return 0;	×
56	if (a.score === 'Infinity')	×
57	return -1;	×
58	if (b.score === 'Infinity')	×
59	return 1;	×
60	return b.score - a.score;	×
61	}
62
63	function substringSpan(sequence : string[], substring : string[]) : [number, number] \| null {
NEW 64	for (let i=0; i < sequence.length; i++) {	×
NEW 65	let found = true;	×
NEW 66	for (let j = 0; j < substring.length; j++) {	×
NEW 67	if (sequence[i+j] !== substring[j]) {	×
NEW 68	found = false;	×
NEW 69	break;	×
70	}
71	}
NEW 72	if (found)	×
NEW 73	return [i, i + substring.length + 1];	×
74	}
NEW 75	return null;	×
76	}
77
78
79	export default class LocalParserClient {	95✔
80	private _locale : string;
81	private _langPack : I18n.LanguagePack;
82	private _tokenizer : I18n.BaseTokenizer;
83	private _predictor : Predictor;
84	private _platform : Tp.BasePlatform\|undefined;
85	private _exactmatcher : ExactMatcher\|undefined;
86	private _tpClient : Tp.BaseClient\|null;
87
88	constructor(modeldir : string,
89	locale : string,
90	platform : Tp.BasePlatform\|undefined,
91	exactmatcher : ExactMatcher\|undefined,
92	tpClient : Tp.BaseClient\|null = null,	6✔
93	options : LocalParserOptions = {}) {	6✔
94	this._locale = locale;	6✔
95	this._langPack = I18n.get(locale);	6✔
96	this._tokenizer = this._langPack.getTokenizer();	6✔
97	this._predictor = new Predictor(modeldir, options);	6✔
98
99	this._platform = platform;	6✔
100	this._exactmatcher = exactmatcher;	6✔
101	this._tpClient = tpClient;	6✔
102	}
103
104	async start() : Promise<void> {
105	await Promise.all([	6✔
106	this._predictor.start(),
107	this._startExactMatcher()
108	]);
109	}
110	async stop() : Promise<void> {
111	await this._predictor.stop();	4✔
112	}
113
114	private async _startExactMatcher() {
115	if (this._exactmatcher)	6!
116	return;	×
117	if (!this._platform \|\| !this._tpClient)	6!
118	return;	6✔
119
120	const prefs = this._platform.getSharedPreferences();	×
121	const developerDir = prefs.get('developer-dir') as string\|string[];	×
122	if (!developerDir)	×
123	return;	×
124	const builder = new ExactMatcherBuilder({	×
125	locale: this._locale,
126	timezone: this._platform.timezone,
127	cachedir: this._platform.getCacheDir(),
128	developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],	×
129	thingpediaClient: this._tpClient,
130	});
131	this._exactmatcher = await builder.load();	×
132	}
133
134	private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]\|undefined) {
135	// only work on contextual
136	if (!contextCode)	4✔
137	return;	2✔
138
139	if (contextCode[0] === '$dialogue' &&	2!
140	contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
141	contextCode[3] === 'sys_anything_else') {
142	for (const prog of programs) {	×
143	if (prog.code[0] === '$dialogue' &&	×
144	prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
145	prog.code[3] === 'cancel')
146	prog.code[3] = 'end';	×
147	}
148	}
149	}
150
151	async sendUtterance(utterance : string,
152	contextCode : string[]\|undefined,
153	contextEntities : EntityMap\|undefined,
154	options : ParseOptions = {}) : Promise<PredictionResult> {	×
155	let tokens : string[], entities : EntityMap;
156	if (options.tokenized) {	4!
157	tokens = utterance.split(' ');	4✔
158	entities = Utils.makeDummyEntities(utterance);	4✔
159	if (contextEntities) {	4!
160	// safety against weird properties
161	for (const key of Object.getOwnPropertyNames(contextEntities)) {	4✔
162	if (/^(.+)_([0-9]+)$/.test(key))	×
163	entities[key] = contextEntities[key];	×
164	}
165	}
166	} else {
167	const tokenized = await this._tokenizer.tokenize(utterance);	×
168	if (contextEntities)	×
169	renumberEntities(tokenized, contextEntities);	×
170	tokens = tokenized.tokens;	×
171	entities = tokenized.entities;	×
172	}
173
174	const answer = options.answer;	4✔
175
176	let result : PredictionCandidate[]\|null = null;	4✔
177	let exact : string[][]\|null = null;	4✔
178	const intent = {	4✔
179	command: 1,
180	other: 0,
181	ignore: 0
182	};
183
184	if (tokens.length === 0) {	4!
185	result = [{	×
186	code: ['$failed', ';'],
187	score: 'Infinity'
188	}];
189	} else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) \|\| tokens[0] === '1' \|\| tokens[0] === '0')) {	4!
190	// if the whole input is just an entity, return that as an answer
191	result = [{	×
192	code: ['$answer', '(', tokens[0], ')', ';'],
193	score: 'Infinity'
194	}];
195	} else if (options.expect === 'MultipleChoice') {	4!
196	const choices = await Promise.all((options.choices \|\| []).map((choice) => this._tokenizer.tokenize(choice)));	×
197	result = choices.map((choice, i) => {	×
198	const distance = editDistance(tokens, choice.tokens);	×
199	return {	×
200	code: ['$choice', '(', String(i), ')', ';'],
201	// distance is between 0 (exact match) and the maximum of the two lengths
202	// adjust distance to be between 0 and 1, higher is better
203	score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
204	};
205	});
206	result.sort(compareScore);	×
207	} else {
208	if (this._exactmatcher)	4!
209	exact = this._exactmatcher.get(tokens);	×
210	}
211
212	if (result === null) {	4!
213	let candidates;
214	if (contextCode)	4✔
215	candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);	2✔
216	else
217	candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);	2✔
218	assert(candidates.length > 0);	4✔
219
220	result = candidates.map((c) => {	4✔
221	const score = c.score.is_correct ?? 1;	4!
222	return {	4✔
223	code: c.answer.split(' '),
224	score: score
225	};
226	});
227
228	intent.other = candidates[0].score.is_ood ?? 0;	4!
229	intent.command = 1 - intent.other;	4✔
230	}
231
232	let result2 = result!; // guaranteed not null	4✔
233	if (exact !== null)	4!
234	result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);	×
235
236	this._applyPostHeuristics(result2, contextCode);	4✔
237
238	if (!options.skip_typechecking) {	4!
239	const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);	×
240
241	result2 = (await Promise.all(result2.map(async (c) => {	×
242	const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {	×
243	timezone: this._platform?.timezone,	×
244	thingpediaClient: this._tpClient,
245	schemaRetriever: schemas
246	});
247
248	if (parsed) {	×
249	return {	×
250	code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
251	locale: this._locale,
252	timezone: this._platform?.timezone,	×
253	compatibility: options.thingtalk_version,
254	ignoreSentence: true
255	}),
256	score: c.score
257	};
258	} else {
259	return null;	×
260	}
261	}))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);	×
262	}
263
264	return {	4✔
265	result: 'ok',
266	tokens: tokens,
267	candidates: result2,
268	entities: entities,
269	intent
270	};
271	}
272
273	async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
274	const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);	×
275	return candidates.map((cand) => {	×
276	return {	×
277	answer: cand.answer,
278	score: cand.score.confidence ?? 1	×
279	};
280	});
281	}
282
283	async translateUtterance(input : string[], contextEntities : EntityMap\|undefined, translationOptions : Record<string, any>) : Promise<GenerationResult[]> {
NEW 284	if (contextEntities) {	×
NEW 285	const allEntities = Object.keys(contextEntities).map((ent) => ent.split(' '));	×
NEW 286	for (const entity of allEntities) {	×
NEW 287	const span = substringSpan(input, entity);	×
NEW 288	if (span) {	×
NEW 289	input.splice(span[0], 0, '"');	×
NEW 290	input.splice(span[1], 0, '"');	×
291	}
292	}
293	}
NEW 294	const candidates = await this._predictor.predict('', input.join(' '), undefined, Translation_TASK, 'id-null', translationOptions);	×
NEW 295	return candidates.map((cand) => {	×
NEW 296	return {	×
297	answer: cand.answer,
298	score: cand.score.confidence ?? 1	×
299	};
300	});
301	}
302	}

stanford-oval / genie-toolkit / 2394

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous