2398

pending completion

Build # 2398

Build Type

Pull #806

travis-ci-com

Committed by

web-flow

Commit Message

Merge 367e922d8 into 7b1a8f0e5

Pull Request Pull Request #806: Wip/translation api

Run Details

10158 of 20584 branches covered (49.35%)

Branch coverage included in aggregate %.

14 of 21 new or added lines in 3 files covered. (66.67%)

117 existing lines in 5 files now uncovered.

24329 of 40787 relevant lines covered (59.65%)

338848.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.27

/lib/prediction/localparserclient.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import * as Tp from 'thingpedia';
import * as ThingTalk from 'thingtalk';

import * as Utils from '../utils/misc-utils';
import * as I18n from '../i18n';
import editDistance from '../utils/edit-distance';
import { EntityMap, renumberEntities } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';

import Predictor from './predictor';
import ExactMatcherBuilder from './exactbuilder';
import {
    ParseOptions,
    PredictionCandidate,
    PredictionResult,
    GenerationResult,
    ExactMatcher
} from './types';

const SEMANTIC_PARSING_TASK = 'almond';
const NLU_TASK = 'almond_dialogue_nlu';
const NLG_TASK = 'almond_dialogue_nlg';
const TRANSLATION_TASK = 'almond_translate';
const NLG_QUESTION = 'what should the agent say ?';

export interface LocalParserOptions {
    id ?: string;
    minibatchSize ?: number;
    maxLatency ?: number;
}

function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
    if (a.score === b.score)
        return 0;
    if (a.score === 'Infinity')
        return -1;
    if (b.score === 'Infinity')
        return 1;
    return b.score - a.score;
}

export default class LocalParserClient {
    private _locale : string;
    private _langPack : I18n.LanguagePack;
    private _tokenizer : I18n.BaseTokenizer;
    private _predictor : Predictor;
    private _platform : Tp.BasePlatform|undefined;
    private _exactmatcher : ExactMatcher|undefined;
    private _tpClient : Tp.BaseClient|null;

    constructor(modeldir : string,
                locale : string,
                platform : Tp.BasePlatform|undefined,
                exactmatcher : ExactMatcher|undefined,
                tpClient : Tp.BaseClient|null = null,
                options : LocalParserOptions = {}) {
        this._locale = locale;
        this._langPack = I18n.get(locale);
        this._tokenizer = this._langPack.getTokenizer();
        this._predictor = new Predictor(modeldir, options);

        this._platform = platform;
        this._exactmatcher = exactmatcher;
        this._tpClient = tpClient;
    }

    async start() : Promise<void> {
        await Promise.all([
            this._predictor.start(),
            this._startExactMatcher()
        ]);
    }
    async stop() : Promise<void> {
        await this._predictor.stop();
    }

    private async _startExactMatcher() {
        if (this._exactmatcher)
            return;
        if (!this._platform || !this._tpClient)
            return;

        const prefs = this._platform.getSharedPreferences();
        const developerDir = prefs.get('developer-dir') as string|string[];
        if (!developerDir)
            return;
        const builder = new ExactMatcherBuilder({
            locale: this._locale,
            timezone: this._platform.timezone,
            cachedir: this._platform.getCacheDir(),
            developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],
            thingpediaClient: this._tpClient,
        });
        this._exactmatcher = await builder.load();
    }

    private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]|undefined) {
        // only work on contextual
        if (!contextCode)
            return;

        if (contextCode[0] === '$dialogue' &&
            contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
            contextCode[3] === 'sys_anything_else') {
            for (const prog of programs) {
                if (prog.code[0] === '$dialogue' &&
                    prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
                    prog.code[3] === 'cancel')
                    prog.code[3] = 'end';
            }
        }
    }

    async sendUtterance(utterance : string,
                        contextCode : string[]|undefined,
                        contextEntities : EntityMap|undefined,
                        options : ParseOptions = {}) : Promise<PredictionResult> {
        let tokens : string[], entities : EntityMap;
        if (options.tokenized) {
            tokens = utterance.split(' ');
            entities = Utils.makeDummyEntities(utterance);
            if (contextEntities) {
                // safety against weird properties
                for (const key of Object.getOwnPropertyNames(contextEntities)) {
                    if (/^(.+)_([0-9]+)$/.test(key))
                        entities[key] = contextEntities[key];
                }
            }
        } else {
            const tokenized = await this._tokenizer.tokenize(utterance);
            if (contextEntities)
                renumberEntities(tokenized, contextEntities);
            tokens = tokenized.tokens;
            entities = tokenized.entities;
        }

        const answer = options.answer;

        let result : PredictionCandidate[]|null = null;
        let exact : string[][]|null = null;
        const intent = {
            command: 1,
            other: 0,
            ignore: 0
        };

        if (tokens.length === 0) {
            result = [{
                code: ['$failed', ';'],
                score: 'Infinity'
            }];
        } else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) || tokens[0] === '1' || tokens[0] === '0')) {
            // if the whole input is just an entity, return that as an answer
            result = [{
                code: ['$answer', '(', tokens[0], ')', ';'],
                score: 'Infinity'
            }];
        } else if (options.expect === 'MultipleChoice') {
            const choices = await Promise.all((options.choices || []).map((choice) => this._tokenizer.tokenize(choice)));
            result = choices.map((choice, i) => {
                const distance = editDistance(tokens, choice.tokens);
                return {
                    code: ['$choice', '(', String(i), ')', ';'],
                    // distance is between 0 (exact match) and the maximum of the two lengths
                    // adjust distance to be between 0 and 1, higher is better
                    score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
                };
            });
            result.sort(compareScore);
        } else {
            if (this._exactmatcher)
                exact = this._exactmatcher.get(tokens);
        }

        if (result === null) {
            let candidates;
            if (contextCode)
                candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);
            else
                candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);
            assert(candidates.length > 0);

            result = candidates.map((c) => {
                const score = c.score.is_correct ?? 1;
                return {
                    code: c.answer.split(' '),
                    score: score
                };
            });

            intent.other = candidates[0].score.is_ood ?? 0;
            intent.command = 1 - intent.other;
        }

        let result2 = result!; // guaranteed not null
        if (exact !== null)
            result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);

        this._applyPostHeuristics(result2, contextCode);

        if (!options.skip_typechecking) {
            const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);

            result2 = (await Promise.all(result2.map(async (c) => {
                const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {
                    timezone: this._platform?.timezone,
                    thingpediaClient: this._tpClient,
                    schemaRetriever: schemas
                });

                if (parsed) {
                    return {
                        code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
                            locale: this._locale,
                            timezone: this._platform?.timezone,
                            compatibility: options.thingtalk_version,
                            ignoreSentence: true
                        }),
                        score: c.score
                    };
                } else {
                    return null;
                }
            }))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);
        }

        return {
            result: 'ok',
            tokens: tokens,
            candidates: result2,
            entities: entities,
            intent
        };
    }

    async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
        const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }

    async translateUtterance(input : string[], contextEntities : EntityMap|undefined, translationOptions : Record<string, unknown>) : Promise<GenerationResult[]> {
        input = Utils.qpisEntities(input, contextEntities);
        const candidates = await this._predictor.predict('', input.join(' '), undefined, TRANSLATION_TASK, 'id-null', translationOptions);
        return candidates.map((cand) => {
            return {
                answer: cand.answer,
                score: cand.score.confidence ?? 1
            };
        });
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import assert from 'assert';	95✔
22	import * as Tp from 'thingpedia';
23	import * as ThingTalk from 'thingtalk';	95✔
24
25	import * as Utils from '../utils/misc-utils';	95✔
26	import * as I18n from '../i18n';	95✔
27	import editDistance from '../utils/edit-distance';	95✔
28	import { EntityMap, renumberEntities } from '../utils/entity-utils';	95✔
29	import * as ThingTalkUtils from '../utils/thingtalk';	95✔
30
31	import Predictor from './predictor';	95✔
32	import ExactMatcherBuilder from './exactbuilder';	95✔
33	import {
34	ParseOptions,
35	PredictionCandidate,
36	PredictionResult,
37	GenerationResult,
38	ExactMatcher
39	} from './types';
40
41	const SEMANTIC_PARSING_TASK = 'almond';	95✔
42	const NLU_TASK = 'almond_dialogue_nlu';	95✔
43	const NLG_TASK = 'almond_dialogue_nlg';	95✔
44	const TRANSLATION_TASK = 'almond_translate';	95✔
45	const NLG_QUESTION = 'what should the agent say ?';	95✔
46
47	export interface LocalParserOptions {
48	id ?: string;
49	minibatchSize ?: number;
50	maxLatency ?: number;
51	}
52
53	function compareScore(a : PredictionCandidate, b : PredictionCandidate) : number {
54	if (a.score === b.score)	×
55	return 0;	×
56	if (a.score === 'Infinity')	×
57	return -1;	×
58	if (b.score === 'Infinity')	×
59	return 1;	×
60	return b.score - a.score;	×
61	}
62
63	export default class LocalParserClient {	95✔
64	private _locale : string;
65	private _langPack : I18n.LanguagePack;
66	private _tokenizer : I18n.BaseTokenizer;
67	private _predictor : Predictor;
68	private _platform : Tp.BasePlatform\|undefined;
69	private _exactmatcher : ExactMatcher\|undefined;
70	private _tpClient : Tp.BaseClient\|null;
71
72	constructor(modeldir : string,
73	locale : string,
74	platform : Tp.BasePlatform\|undefined,
75	exactmatcher : ExactMatcher\|undefined,
76	tpClient : Tp.BaseClient\|null = null,	6✔
77	options : LocalParserOptions = {}) {	6✔
78	this._locale = locale;	6✔
79	this._langPack = I18n.get(locale);	6✔
80	this._tokenizer = this._langPack.getTokenizer();	6✔
81	this._predictor = new Predictor(modeldir, options);	6✔
82
83	this._platform = platform;	6✔
84	this._exactmatcher = exactmatcher;	6✔
85	this._tpClient = tpClient;	6✔
86	}
87
88	async start() : Promise<void> {
89	await Promise.all([	6✔
90	this._predictor.start(),
91	this._startExactMatcher()
92	]);
93	}
94	async stop() : Promise<void> {
95	await this._predictor.stop();	4✔
96	}
97
98	private async _startExactMatcher() {
99	if (this._exactmatcher)	6!
UNCOV 100	return;	×
101	if (!this._platform \|\| !this._tpClient)	6!
102	return;	6✔
103
UNCOV 104	const prefs = this._platform.getSharedPreferences();	×
UNCOV 105	const developerDir = prefs.get('developer-dir') as string\|string[];	×
UNCOV 106	if (!developerDir)	×
UNCOV 107	return;	×
UNCOV 108	const builder = new ExactMatcherBuilder({	×
109	locale: this._locale,
110	timezone: this._platform.timezone,
111	cachedir: this._platform.getCacheDir(),
112	developerdir: Array.isArray(developerDir) ? developerDir : [developerDir],	×
113	thingpediaClient: this._tpClient,
114	});
UNCOV 115	this._exactmatcher = await builder.load();	×
116	}
117
118	private _applyPostHeuristics(programs : PredictionCandidate[], contextCode : string[]\|undefined) {
119	// only work on contextual
120	if (!contextCode)	4✔
121	return;	2✔
122
123	if (contextCode[0] === '$dialogue' &&	2!
124	contextCode[1] === '@org.thingpedia.dialogue.transaction' &&
125	contextCode[3] === 'sys_anything_else') {
UNCOV 126	for (const prog of programs) {	×
UNCOV 127	if (prog.code[0] === '$dialogue' &&	×
128	prog.code[1] === '@org.thingpedia.dialogue.transaction' &&
129	prog.code[3] === 'cancel')
UNCOV 130	prog.code[3] = 'end';	×
131	}
132	}
133	}
134
135	async sendUtterance(utterance : string,
136	contextCode : string[]\|undefined,
137	contextEntities : EntityMap\|undefined,
138	options : ParseOptions = {}) : Promise<PredictionResult> {	×
139	let tokens : string[], entities : EntityMap;
140	if (options.tokenized) {	4!
141	tokens = utterance.split(' ');	4✔
142	entities = Utils.makeDummyEntities(utterance);	4✔
143	if (contextEntities) {	4!
144	// safety against weird properties
145	for (const key of Object.getOwnPropertyNames(contextEntities)) {	4✔
146	if (/^(.+)_([0-9]+)$/.test(key))	×
UNCOV 147	entities[key] = contextEntities[key];	×
148	}
149	}
150	} else {
UNCOV 151	const tokenized = await this._tokenizer.tokenize(utterance);	×
UNCOV 152	if (contextEntities)	×
UNCOV 153	renumberEntities(tokenized, contextEntities);	×
UNCOV 154	tokens = tokenized.tokens;	×
UNCOV 155	entities = tokenized.entities;	×
156	}
157
158	const answer = options.answer;	4✔
159
160	let result : PredictionCandidate[]\|null = null;	4✔
161	let exact : string[][]\|null = null;	4✔
162	const intent = {	4✔
163	command: 1,
164	other: 0,
165	ignore: 0
166	};
167
168	if (tokens.length === 0) {	4!
169	result = [{	×
170	code: ['$failed', ';'],
171	score: 'Infinity'
172	}];
173	} else if (tokens.length === 1 && (/^[A-Z]/.test(tokens[0]) \|\| tokens[0] === '1' \|\| tokens[0] === '0')) {	4!
174	// if the whole input is just an entity, return that as an answer
UNCOV 175	result = [{	×
176	code: ['$answer', '(', tokens[0], ')', ';'],
177	score: 'Infinity'
178	}];
179	} else if (options.expect === 'MultipleChoice') {	4!
UNCOV 180	const choices = await Promise.all((options.choices \|\| []).map((choice) => this._tokenizer.tokenize(choice)));	×
UNCOV 181	result = choices.map((choice, i) => {	×
UNCOV 182	const distance = editDistance(tokens, choice.tokens);	×
UNCOV 183	return {	×
184	code: ['$choice', '(', String(i), ')', ';'],
185	// distance is between 0 (exact match) and the maximum of the two lengths
186	// adjust distance to be between 0 and 1, higher is better
187	score: 1 - distance / Math.max(tokens.length, choice.tokens.length)
188	};
189	});
UNCOV 190	result.sort(compareScore);	×
191	} else {
192	if (this._exactmatcher)	4!
UNCOV 193	exact = this._exactmatcher.get(tokens);	×
194	}
195
196	if (result === null) {	4!
197	let candidates;
198	if (contextCode)	4✔
199	candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), answer, NLU_TASK, options.example_id);	2✔
200	else
201	candidates = await this._predictor.predict(tokens.join(' '), undefined, answer, SEMANTIC_PARSING_TASK, options.example_id);	2✔
202	assert(candidates.length > 0);	4✔
203
204	result = candidates.map((c) => {	4✔
205	const score = c.score.is_correct ?? 1;	4!
206	return {	4✔
207	code: c.answer.split(' '),
208	score: score
209	};
210	});
211
212	intent.other = candidates[0].score.is_ood ?? 0;	4!
213	intent.command = 1 - intent.other;	4✔
214	}
215
216	let result2 = result!; // guaranteed not null	4✔
217	if (exact !== null)	4!
UNCOV 218	result2 = exact.map((code) : PredictionCandidate => ({ code, score: 'Infinity' })).concat(result2);	×
219
220	this._applyPostHeuristics(result2, contextCode);	4✔
221
222	if (!options.skip_typechecking) {	4!
UNCOV 223	const schemas = new ThingTalk.SchemaRetriever(this._tpClient!, null, true);	×
224
UNCOV 225	result2 = (await Promise.all(result2.map(async (c) => {	×
UNCOV 226	const parsed = await ThingTalkUtils.parsePrediction(c.code, entities, {	×
227	timezone: this._platform?.timezone,	×
228	thingpediaClient: this._tpClient,
229	schemaRetriever: schemas
230	});
231
UNCOV 232	if (parsed) {	×
UNCOV 233	return {	×
234	code: ThingTalkUtils.serializePrediction(parsed, tokens, entities, {
235	locale: this._locale,
236	timezone: this._platform?.timezone,	×
237	compatibility: options.thingtalk_version,
238	ignoreSentence: true
239	}),
240	score: c.score
241	};
242	} else {
UNCOV 243	return null;	×
244	}
UNCOV 245	}))).filter(<T>(c : T) : c is Exclude<T, null> => c !== null);	×
246	}
247
248	return {	4✔
249	result: 'ok',
250	tokens: tokens,
251	candidates: result2,
252	entities: entities,
253	intent
254	};
255	}
256
257	async generateUtterance(contextCode : string[], contextEntities : EntityMap, targetAct : string[]) : Promise<GenerationResult[]> {
UNCOV 258	const candidates = await this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, undefined, NLG_TASK);	×
259	return candidates.map((cand) => {	×
UNCOV 260	return {	×
261	answer: cand.answer,
262	score: cand.score.confidence ?? 1	×
263	};
264	});
265	}
266
267	async translateUtterance(input : string[], contextEntities : EntityMap\|undefined, translationOptions : Record<string, unknown>) : Promise<GenerationResult[]> {
UNCOV 268	input = Utils.qpisEntities(input, contextEntities);	×
UNCOV 269	const candidates = await this._predictor.predict('', input.join(' '), undefined, TRANSLATION_TASK, 'id-null', translationOptions);	×
UNCOV 270	return candidates.map((cand) => {	×
UNCOV 271	return {	×
272	answer: cand.answer,
273	score: cand.score.confidence ?? 1	×
274	};
275	});
276	}
277	}

stanford-oval / genie-toolkit / 2398

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous