2416

pending completion

Build # 2416

Build Type

Pull #806

travis-ci-com

Committed by

web-flow

Commit Message

Merge 417d7d2d9 into 586a22741

Pull Request Pull Request #806: Wip/translation api

Run Details

10088 of 20444 branches covered (49.34%)

Branch coverage included in aggregate %.

13 of 62 new or added lines in 5 files covered. (20.97%)

2 existing lines in 1 file now uncovered.

24258 of 40615 relevant lines covered (59.73%)

375835.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.27

/lib/prediction/localparserclient.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import * as Tp from 'thingpedia';
import * as ThingTalk from 'thingtalk';

import * as Utils from '../utils/misc-utils';
import * as I18n from '../i18n';
import editDistance from '../utils/edit-distance';
import { EntityMap, renumberEntities } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';

import Predictor from './predictor';
import ExactMatcherBuilder from './exactbuilder';
import {
    ParseOptions,
    PredictionCandidate,
    PredictionResult,
    GenerationResult,
    ExactMatcher,
    GenerationOptions
} from './types';

const SEMANTIC_PARSING_TASK = 'almond';
const NLU_TASK = 'almond_dialogue_nlu';
const NLG_TASK = 'almond_dialogue_nlg';
const TRANSLATION_TASK = 'almond_translate';
const NLG_QUESTION = 'what should the agent say ?';

export interface LocalParserOptions {
    id ?: string;
    minibatchSize ?: number;
    maxLatency ?: number;
}

function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
    if (a.score === b.score)
        return 0;
    if (a.score === 'Infinity')
        return -1;
    if (b.score === 'Infinity')
        return 1;
    return b.score - a.score;
}

export default class LocalParserClient {
    private _locale : string;
    private _langPack : I18n.LanguagePack;
    private _tokenizer : I18n.BaseTokenizer;
    private _predictor : Predictor;
    private _platform : Tp.BasePlatform|undefined;
    private _exactmatcher : ExactMatcher|undefined;
    private _tpClient : Tp.BaseClient|null;

    constructor(modeldir : string,
                locale : string,
                platform : Tp.BasePlatform|undefined,
                exactmatcher : ExactMatcher|undefined,
                tpClient : Tp.BaseClient|null = null,
                options : LocalParserOptions = {}) {
        this._locale = locale;
        this._langPack = I18n.get(locale);
        this._tokenizer = this._langPack.getTokenizer();
        this._predictor = new Predictor(modeldir, options);

        this._platform = platform;
        this._exactmatcher = exactmatcher;
        this._tpClient = tpClient;
    }

    async start() : Promise<void> {
        await Promise.all([
            this._predictor.start(),
            this._startExactMatcher()
        ]);
    }
    async stop() : Promise<void> {
        await this._predictor.stop();
    }

    private async _startExactMatcher() {
        if (this._exactmatcher)
            return;
        if (!this._platform || !this._tpClient)
            return;

        const prefs = this._platform.getSharedPreferences();
        const developerDir = prefs.get('developer-dir') as string|string[];
        if (!developerDir)
            return;
        const builder = new ExactMatcherBuilder({
            locale: this._locale,
            timezone: this._platform.timezone,
            cachedir: this._platform.getCacheDir(),
            developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],
            thingpediaClient: this._tpClient,
        });
        this._exactmatcher = await builder.load();
    }

    private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]|undefined) {
        // only work on contextual
        if (!contextCode)
            return;

        if (contextCode[0] === '$dialogue' &&
            contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
            contextCode[3] === 'sys_anything_else') {
            for (const prog of programs) {
                if (prog.code[0] === '$dialogue' &&
                    prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
                    prog.code[3] === 'cancel')
                    prog.code[3] = 'end';
            }
        }
    }

    async sendUtterance(utterance : string,
                        contextCode : string[]|undefined,
                        contextEntities : EntityMap|undefined,
                        options : ParseOptions = {}) : Promise<PredictionResult> {
        let tokens : string[], entities : EntityMap;
        if (options.tokenized) {
            tokens = utterance.split(' ');
            entities = Utils.makeDummyEntities(utterance);
            if (contextEntities) {
                // safety against weird properties
                for (const key of Object.getOwnPropertyNames(contextEntities)) {
                    if (/^(.+)_([0-9]+)$/.test(key))
                        entities[key] = contextEntities[key];
                }
            }
        } else {
            const tokenized = await this._tokenizer.tokenize(utterance);
            if (contextEntities)
                renumberEntities(tokenized, contextEntities);
            tokens = tokenized.tokens;
            entities = tokenized.entities;
        }

        const answer = options.answer;

        let result : PredictionCandidate[]|null = null;
        let exact : string[][]|null = null;
        const intent = {
            command: 1,
            other: 0,
            ignore: 0
        };

        if (tokens.length === 0) {
            result = [{
                code: ['$failed', ';'],
                score: 'Infinity'
            }];
        } else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) || tokens[0] === '1' || tokens[0] === '0')) {
            // if the whole input is just an entity, return that as an answer
            result = [{
                code: ['$answer', '(', tokens[0], ')', ';'],
                score: 'Infinity'
            }];
        } else if (options.expect === 'MultipleChoice') {
            const choices = await Promise.all((options.choices || []).map((choice) => this._tokenizer.tokenize(choice)));
            result = choices.map((choice, i) => {
                const distance = editDistance(tokens, choice.tokens);
                return {
                    code: ['$choice', '(', String(i), ')', ';'],
                    // distance is between 0 (exact match) and the maximum of the two lengths
                    // adjust distance to be between 0 and 1, higher is better
                    score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
                };
            });
            result.sort(compareScore);
        } else {
            if (this._exactmatcher)
                exact = this._exactmatcher.get(tokens);
        }

        if (result === null) {
            let candidates;
            if (contextCode)
                candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);
            else
                candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);
            assert(candidates.length > 0);

            result = candidates.map((c) => {
                const score = c.score.is_correct ?? 1;
                return {
                    code: c.answer.split(' '),
                    score: score
                };
            });

            intent.other = candidates[0].score.is_ood ?? 0;
            intent.command = 1 - intent.other;
        }

        let result2 = result!; // guaranteed not null
        if (exact !== null)
            result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);

        this._applyPostHeuristics(result2, contextCode);

        if (!options.skip_typechecking) {
            const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);

            result2 = (await Promise.all(result2.map(async (c) => {
                const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {
                    timezone: this._platform?.timezone,
                    thingpediaClient: this._tpClient,
                    schemaRetriever: schemas
                });

                if (parsed) {
                    return {
                        code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
                            locale: this._locale,
                            timezone: this._platform?.timezone,
                            compatibility: options.thingtalk_version,
                            ignoreSentence: true
                        }),
                        score: c.score
                    };
                } else {
                    return null;
                }
            }))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);
        }

        return {
            result: 'ok',
            tokens: tokens,
            candidates: result2,
            entities: entities,
            intent
        };
    }

    async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
        const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }

    async translateUtterance(input : string[], entities : string[]|undefined, generationOptions : GenerationOptions) : Promise<GenerationResult[]> {
        input = Utils.qpisEntities(input, entities);
        const candidates = await this._predictor.predict('', input.join(' '), undefined, TRANSLATION_TASK, undefined, generationOptions);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import assert from 'assert';	95✔
22	import * as Tp from 'thingpedia';
23	import * as ThingTalk from 'thingtalk';	95✔
24
25	import * as Utils from '../utils/misc-utils';	95✔
26	import * as I18n from '../i18n';	95✔
27	import editDistance from '../utils/edit-distance';	95✔
28	import { EntityMap, renumberEntities } from '../utils/entity-utils';	95✔
29	import * as ThingTalkUtils from '../utils/thingtalk';	95✔
30
31	import Predictor from './predictor';	95✔
32	import ExactMatcherBuilder from './exactbuilder';	95✔
33	import {
34	ParseOptions,
35	PredictionCandidate,
36	PredictionResult,
37	GenerationResult,
38	ExactMatcher,
39	GenerationOptions
40	} from './types';
41
42	const SEMANTIC_PARSING_TASK = 'almond';	95✔
43	const NLU_TASK = 'almond_dialogue_nlu';	95✔
44	const NLG_TASK = 'almond_dialogue_nlg';	95✔
45	const TRANSLATION_TASK = 'almond_translate';	95✔
46	const NLG_QUESTION = 'what should the agent say ?';	95✔
47
48	export interface LocalParserOptions {
49	id ?: string;
50	minibatchSize ?: number;
51	maxLatency ?: number;
52	}
53
54	function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
55	if (a.score === b.score)	×
56	return 0;	×
57	if (a.score === 'Infinity')	×
58	return -1;	×
59	if (b.score === 'Infinity')	×
60	return 1;	×
61	return b.score - a.score;	×
62	}
63
64	export default class LocalParserClient {	95✔
65	private _locale : string;
66	private _langPack : I18n.LanguagePack;
67	private _tokenizer : I18n.BaseTokenizer;
68	private _predictor : Predictor;
69	private _platform : Tp.BasePlatform\|undefined;
70	private _exactmatcher : ExactMatcher\|undefined;
71	private _tpClient : Tp.BaseClient\|null;
72
73	constructor(modeldir : string,
74	locale : string,
75	platform : Tp.BasePlatform\|undefined,
76	exactmatcher : ExactMatcher\|undefined,
77	tpClient : Tp.BaseClient\|null = null,	6✔
78	options : LocalParserOptions = {}) {	6✔
79	this._locale = locale;	6✔
80	this._langPack = I18n.get(locale);	6✔
81	this._tokenizer = this._langPack.getTokenizer();	6✔
82	this._predictor = new Predictor(modeldir, options);	6✔
83
84	this._platform = platform;	6✔
85	this._exactmatcher = exactmatcher;	6✔
86	this._tpClient = tpClient;	6✔
87	}
88
89	async start() : Promise<void> {
90	await Promise.all([	6✔
91	this._predictor.start(),
92	this._startExactMatcher()
93	]);
94	}
95	async stop() : Promise<void> {
96	await this._predictor.stop();	4✔
97	}
98
99	private async _startExactMatcher() {
100	if (this._exactmatcher)	6!
101	return;	×
102	if (!this._platform \|\| !this._tpClient)	6!
103	return;	6✔
104
105	const prefs = this._platform.getSharedPreferences();	×
106	const developerDir = prefs.get('developer-dir') as string\|string[];	×
107	if (!developerDir)	×
108	return;	×
109	const builder = new ExactMatcherBuilder({	×
110	locale: this._locale,
111	timezone: this._platform.timezone,
112	cachedir: this._platform.getCacheDir(),
113	developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],	×
114	thingpediaClient: this._tpClient,
115	});
116	this._exactmatcher = await builder.load();	×
117	}
118
119	private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]\|undefined) {
120	// only work on contextual
121	if (!contextCode)	4✔
122	return;	2✔
123
124	if (contextCode[0] === '$dialogue' &&	2!
125	contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
126	contextCode[3] === 'sys_anything_else') {
127	for (const prog of programs) {	×
128	if (prog.code[0] === '$dialogue' &&	×
129	prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
130	prog.code[3] === 'cancel')
131	prog.code[3] = 'end';	×
132	}
133	}
134	}
135
136	async sendUtterance(utterance : string,
137	contextCode : string[]\|undefined,
138	contextEntities : EntityMap\|undefined,
139	options : ParseOptions = {}) : Promise<PredictionResult> {	×
140	let tokens : string[], entities : EntityMap;
141	if (options.tokenized) {	4!
142	tokens = utterance.split(' ');	4✔
143	entities = Utils.makeDummyEntities(utterance);	4✔
144	if (contextEntities) {	4!
145	// safety against weird properties
146	for (const key of Object.getOwnPropertyNames(contextEntities)) {	4✔
147	if (/^(.+)_([0-9]+)$/.test(key))	×
148	entities[key] = contextEntities[key];	×
149	}
150	}
151	} else {
152	const tokenized = await this._tokenizer.tokenize(utterance);	×
153	if (contextEntities)	×
154	renumberEntities(tokenized, contextEntities);	×
155	tokens = tokenized.tokens;	×
156	entities = tokenized.entities;	×
157	}
158
159	const answer = options.answer;	4✔
160
161	let result : PredictionCandidate[]\|null = null;	4✔
162	let exact : string[][]\|null = null;	4✔
163	const intent = {	4✔
164	command: 1,
165	other: 0,
166	ignore: 0
167	};
168
169	if (tokens.length === 0) {	4!
170	result = [{	×
171	code: ['$failed', ';'],
172	score: 'Infinity'
173	}];
174	} else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) \|\| tokens[0] === '1' \|\| tokens[0] === '0')) {	4!
175	// if the whole input is just an entity, return that as an answer
176	result = [{	×
177	code: ['$answer', '(', tokens[0], ')', ';'],
178	score: 'Infinity'
179	}];
180	} else if (options.expect === 'MultipleChoice') {	4!
181	const choices = await Promise.all((options.choices \|\| []).map((choice) => this._tokenizer.tokenize(choice)));	×
182	result = choices.map((choice, i) => {	×
183	const distance = editDistance(tokens, choice.tokens);	×
184	return {	×
185	code: ['$choice', '(', String(i), ')', ';'],
186	// distance is between 0 (exact match) and the maximum of the two lengths
187	// adjust distance to be between 0 and 1, higher is better
188	score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
189	};
190	});
191	result.sort(compareScore);	×
192	} else {
193	if (this._exactmatcher)	4!
194	exact = this._exactmatcher.get(tokens);	×
195	}
196
197	if (result === null) {	4!
198	let candidates;
199	if (contextCode)	4✔
200	candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);	2✔
201	else
202	candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);	2✔
203	assert(candidates.length > 0);	4✔
204
205	result = candidates.map((c) => {	4✔
206	const score = c.score.is_correct ?? 1;	4!
207	return {	4✔
208	code: c.answer.split(' '),
209	score: score
210	};
211	});
212
213	intent.other = candidates[0].score.is_ood ?? 0;	4!
214	intent.command = 1 - intent.other;	4✔
215	}
216
217	let result2 = result!; // guaranteed not null	4✔
218	if (exact !== null)	4!
219	result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);	×
220
221	this._applyPostHeuristics(result2, contextCode);	4✔
222
223	if (!options.skip_typechecking) {	4!
224	const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);	×
225
226	result2 = (await Promise.all(result2.map(async (c) => {	×
227	const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {	×
228	timezone: this._platform?.timezone,	×
229	thingpediaClient: this._tpClient,
230	schemaRetriever: schemas
231	});
232
233	if (parsed) {	×
234	return {	×
235	code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
236	locale: this._locale,
237	timezone: this._platform?.timezone,	×
238	compatibility: options.thingtalk_version,
239	ignoreSentence: true
240	}),
241	score: c.score
242	};
243	} else {
244	return null;	×
245	}
246	}))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);	×
247	}
248
249	return {	4✔
250	result: 'ok',
251	tokens: tokens,
252	candidates: result2,
253	entities: entities,
254	intent
255	};
256	}
257
258	async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
259	const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);	×
260	return candidates.map((cand) => {	×
261	return {	×
262	answer: cand.answer,
263	score: cand.score.confidence ?? 1	×
264	};
265	});
266	}
267
268	async translateUtterance(input : string[], entities : string[]\|undefined, generationOptions : GenerationOptions) : Promise<GenerationResult[]> {
NEW 269	input = Utils.qpisEntities(input, entities);	×
NEW 270	const candidates = await this._predictor.predict('', input.join(' '), undefined, TRANSLATION_TASK, undefined, generationOptions);	×
NEW 271	return candidates.map((cand) => {	×
NEW 272	return {	×
273	answer: cand.answer,
274	score: cand.score.confidence ?? 1	×
275	};
276	});
277	}
278	}

stanford-oval / genie-toolkit / 2416

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous