1310

pending completion

Build # 1310

Build Type

Pull #342

travis-ci-com

Committed by

web-flow

Commit Message

Merge f84d6f6bf into 6391b9c2f

Pull Request Pull Request #342: Yet another round of fixes

Run Details

6904 of 12985 branches covered (53.17%)

Branch coverage included in aggregate %.

119 of 138 new or added lines in 10 files covered. (86.23%)

16 existing lines in 4 files now uncovered.

15592 of 24838 relevant lines covered (62.77%)

1584650.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.84

/lib/dialogue-agent/dialogue-loop.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const assert = require('assert');

const ThingTalk = require('thingtalk');
const Ast = ThingTalk.Ast;
const interpolate = require('string-interp');
const AsyncQueue = require('consumer-queue');

const { getProgramIcon } = require('../utils/icons');
const ValueCategory = require('./value-category');
const QueueItem = require('./dialogue_queue');
const { CancellationError } = require('./errors');

const Helpers = require('./helpers');
const { showNotification, showError } = require('./notifications');
const { getFallbackExamples } = require('./fallback');
const { computeNewState, prepareContextForPrediction } = require('./dialogue_state_utils');
const DialoguePolicy = require('./dialogue_policy');

const ExecutionDialogueAgent = require('./execution_dialogue_agent');

// TODO: load the policy.yaml file instead
const POLICY_NAME = 'org.thingpedia.dialogue.transaction';
const TERMINAL_STATE = 'sys_end';

module.exports = class DialogueLoop {
    constructor(conversation, engine, debug) {
        this._userInputQueue = new AsyncQueue();
        this._notifyQueue = new AsyncQueue();

        this._debug = debug;
        this.conversation = conversation;
        this.engine = engine;
        this._prefs = engine.platform.getSharedPreferences();
        this.formatter = new ThingTalk.Formatter(engine.platform.locale, engine.platform.timezone, engine.schemas, conversation.gettext);
        this.icon = null;
        this.expecting = null;
        this.platformData = null;
        this._choices = null;

        this._mgrResolve = null;
        this._mgrPromise = null;

        this._agent = new ExecutionDialogueAgent(engine, this, debug);
        this._policy = new DialoguePolicy(this, conversation);
        this._dialogueState = null; // thingtalk dialogue state
        this._executorState = undefined; // private object managed by DialogueExecutor
        this._lastNotificationApp = undefined;
    }

    get _() {
        return this.conversation._;
    }
    get isAnonymous() {
        return this.conversation.isAnonymous;
    }

    debug() {
        if (!this._debug)
            return;
        console.log.apply(console, arguments);
    }

    interpolate(msg, args) {
        return interpolate(msg, args, {
            locale: this.locale,
            timezone: this.timezone
        });
    }

    async nextIntent() {
        await this.conversation.sendAskSpecial();
        this._mgrPromise = null;
        this._mgrResolve();
        const intent = await this._userInputQueue.pop();
        this.platformData = intent.platformData;
        return intent;
    }

    _checkPolicy(policyName) {
        if (policyName !== POLICY_NAME) {
            // TODO we should download the policy from Thingpedia
            throw new Error(`Invalid dialogue policy ${policyName}`);
        }
    }

    async _handleUICommand(intent) {
        switch (intent.type) {
        case 'stop':
            // stop means cancel, but without a failure message
            throw new CancellationError();

        case 'nevermind':
            await this.reply(this._("Sorry I couldn't help on that."));
            throw new CancellationError();

        case 'debug':
            await this.reply("Current State:\n" + this._dialogueState.prettyprint());
            break;

        case 'wakeup':
            // nothing to do
            break;

        default:
            await this.fail();
        }
    }

    async _computePrediction(intent) {
        // handle all intents generated internally and by the UI:
        //
        // - Failed when parsing fails
        // - Answer when the user clicks a button, or when the agent is in "raw mode"
        // - NeverMind when the user clicks the X button
        // - Debug when the user clicks/types "debug"
        // - WakeUp when the user says the wake word and nothing else
        if (intent.isFailed) {
            await getFallbackExamples(this, intent.utterance);
            return null;
        }
        if (intent.isUnsupported) {
            this.icon = null;
            await this.reply(this._("Sorry, I don't know how to do that yet."));
            throw new CancellationError();
        }
        if (intent.isAnswer) {
            const handled = await this._policy.handleAnswer(this._dialogueState, intent.value);
            if (!handled) {
                await this.fail();
                return null;
            }
            return handled;
        }

        if (intent.isProgram) {
            // convert thingtalk programs to dialogue states so we can use "\t" without too much typing
            const prediction = new Ast.DialogueState(null, 'org.thingpedia.dialogue.transaction', 'execute', null, []);
            for (let stmt of intent.program.rules)
                prediction.history.push(new Ast.DialogueHistoryItem(null, stmt, null, 'accepted'));
            return prediction;
        }

        if (intent.isUICommand) {
            await this._handleUICommand(intent);
            return null;
        }

        assert(intent.isDialogueState);
        return intent.prediction;
    }

    _useNeuralNLG() {
        return this._prefs.get('experimental-use-neural-nlg');
    }

    async _doAgentReply() {
        let oldState = this._dialogueState;

        let expect, utterance;
        if (this._useNeuralNLG()) {
            [this._dialogueState, expect] = await this._policy.chooseAction(this._dialogueState);

            const policyPrediction = computeNewState(oldState, this._dialogueState);
            this.debug(`Agent act:`);
            this.debug(policyPrediction.prettyprint());

            const context = prepareContextForPrediction(oldState, 'agent');
            await this.conversation.setContext(context, { allocateEntities: true, typeAnnotations: false });

            utterance = await this.conversation.generateAnswer(policyPrediction);
        } else {
            [this._dialogueState, expect, utterance] = await this._policy.chooseAction(this._dialogueState);
        }

        this.icon = getProgramIcon(this._dialogueState);
        await this.reply(utterance);
        if (this._dialogueState.dialogueAct === TERMINAL_STATE)
            throw new CancellationError();

        await this.setExpected(expect);
        return expect;
    }

    async _handleUserInput(intent) {
        for (;;) {
            const prediction = await this._computePrediction(intent);
            if (prediction === null) {
                intent = await this.nextIntent();
                continue;
            }
            this._dialogueState = computeNewState(this._dialogueState, prediction);
            this._checkPolicy(this._dialogueState.policy);
            this.icon = getProgramIcon(this._dialogueState);

            //this.debug(`Before execution:`);
            //this.debug(this._dialogueState.prettyprint());

            [this._dialogueState, this._executorState] = await this._agent.execute(this._dialogueState, this._executorState);
            this.debug(`Execution state:`);
            this.debug(this._dialogueState.prettyprint());

            const expect = await this._doAgentReply();
            if (expect === null)
                return;

            intent = await this.nextIntent();
        }
    }

    async _handleAPICall(call) {
        if (call.isNotification) {
            await showNotification(this, call.appId, call.icon, call.outputType, call.outputValue, this._lastNotificationApp);
            this._lastNotificationApp = call.appId;
        } else if (call.isError) {
            await showError(this, call.appId, call.icon, call.error, this._lastNotificationApp);
            this._lastNotificationApp = call.appId;
        }
    }

    async _loop(showWelcome) {
        // if we want to show the welcome message, we run the policy on the `null` state, which will return the sys_greet intent
        if (showWelcome) {
            await this._doAgentReply();
            // reset the dialogue state here; if we don't, we we'll see sys_greet as an agent
            // dialogue act; this is never seen in training, because in training the user speaks
            // first, so it confuses the neural network
            this._dialogueState = null;
            // the utterance ends with "what can i do for you?", which is expect = 'generic'
            // but we don't want to keep the microphone open here, we want to go back to wake-word mode
            // so we unconditionally close the round here
            await this.setExpected(null);
        }

        for (;;) {
            const item = await this.nextQueueItem();
            try {
                if (item.isUserInput) {
                    this._lastNotificationApp = undefined;
                    await this._handleUserInput(item.intent);
                } else {
                    await this._handleAPICall(item);
                    this._dialogueState = null;
                }
            } catch(e) {
                if (e.code === 'ECANCELLED') {
                    await this.reset();
                } else {
                    if (item.isUserInput) {
                        await this.replyInterp(this._("Sorry, I had an error processing your command: ${error}."), {//"
                            error: Helpers.formatError(this, e)
                        });
                    } else {
                        await this.replyInterp(this._("Sorry, that did not work: ${error}."), {
                            error: Helpers.formatError(this, e)
                        });
                    }
                    console.error(e);
                }
            }
        }
    }

    get dialogueState() {
        return this._dialogueState;
    }

    set dialogueState(newState) {
        this._dialogueState = newState;
    }

    async nextQueueItem() {
        this.setExpected(null);
        await this.conversation.sendAskSpecial();
        this._mgrPromise = null;
        this._mgrResolve();
        const queueItem = await this._notifyQueue.pop();
        if (queueItem.isUserInput)
            this.platformData = queueItem.intent.platformData;
        else
            this.platformData = {};
        return queueItem;
    }

    async lookingFor() {
        const ALLOWED_UNITS = {
            'ms': ['ms', 's', 'min', 'h', 'day', 'week', 'mon', 'year'],
            'm': ['m', 'km', 'mm', 'cm', 'mi', 'in'],
            'mps': ['mps', 'kmph', 'mph'],
            'kg': ['kg', 'g', 'lb', 'oz'],
            'Pa': ['Pa', 'bar', 'psi', 'mmHg', 'inHg', 'atm'],
            'C': ['C', 'F', 'K'],
            'kcal': ['kcal', 'kJ'],
            'byte': ['byte', 'KB', 'KiB', 'MB', 'MiB', 'GB', 'GiB', 'TB', 'TiB']
        };

        if (this.expecting === null) {
            await this.reply(this._("In fact, I did not ask for anything at all!"));
        } else if (this.expecting === ValueCategory.YesNo) {
            await this.reply(this._("Sorry, I need you to confirm the last question first."));
        } else if (this.expecting === ValueCategory.MultipleChoice) {
            await this.reply(this._("Could you choose one of the following?"));
            this.conversation.resendChoices();
        } else if (this.expecting.isMeasure) {
            await this.replyInterp(this._("I'm looking for ${expecting:select:\
                ms {a time interval}\
                m {a length}\
                mps {a speed}\
                kg {a weight}\
                Pa {a pressure}\
                C {a temperature}\
                kcal {an energy}\
                byte {a size}\
                other {a value}\
            } in any of the supported units (${units})."), {
                expecting: this.expecting.unit,
                units: ALLOWED_UNITS[this.expecting.unit]
            });
        } else if (this.expecting === ValueCategory.Number) {
            await this.reply(this._("Could you give me a number?"));
        } else if (this.expecting === ValueCategory.Date) {
            await this.reply(this._("Could you give me a date?"));
        } else if (this.expecting === ValueCategory.Time) {
            await this.reply(this._("Could you give me a time of day?"));
        } else if (this.expecting === ValueCategory.Picture) {
            await this.reply(this._("Could you upload a picture?"));
        } else if (this.expecting === ValueCategory.Location) {
            await this.reply(this._("Could you give me a place?"));
        } else if (this.expecting === ValueCategory.PhoneNumber) {
            await this.reply(this._("Could you give me a phone number?"));
        } else if (this.expecting === ValueCategory.EmailAddress) {
            await this.reply(this._("Could you give me an email address?"));
        } else if (this.expecting === ValueCategory.RawString || this.expecting === ValueCategory.Password) {
            // ValueCategory.RawString puts Almond in raw mode,
            // so we accept almost everything
            // but this will happen if the user clicks a button
            // or upload a picture
            await this.reply(this._("Which is interesting, because I'll take anything at all. Just type your mind!"));
        } else if (this.expecting === ValueCategory.Command) {
            await this.reply(this._("I'm looking for a command."));
        } else if (this.expecting === ValueCategory.Predicate) {
            await this.reply(this._("I'm looking for a filter"));
        } else {
            await this.reply(this._("In fact, I'm not even sure what I asked. Sorry!"));
        }
    }

    async fail(msg) {
        if (this.expecting === null) {
            if (msg) {
                await this.replyInterp(this._("Sorry, I did not understand that: ${error}. Can you rephrase it?"), {
                    error: msg
                });
            } else {
                await this.reply(this._("Sorry, I did not understand that. Can you rephrase it?"));
            }
        } else {
            if (msg)
                await this.replyInterp(this._("Sorry, I did not understand that: ${error}."), { error: msg });
            else
                await this.reply(this._("Sorry, I did not understand that."));
            await this.lookingFor();
        }
        return true;
    }

    setExpected(expected) {
        if (expected === undefined)
            throw new TypeError();
        this.expecting = expected;
        const context = prepareContextForPrediction(this._dialogueState, 'user');
        this.conversation.setContext(context, { typeAnnotations: false });
        this.conversation.expect(expected);
    }

    /**
     * Ask a question to the user.
     *
     * This is a legacy method used for certain scripted interactions.
     */
    async ask(expected, question, args) {
        await this.replyInterp(question, args);
        await this.setExpected(expected);
        let intent = await this.nextIntent();
        while (!intent.isAnswer || intent.category !== expected) {
            if (intent.isUICommand)
                await this._handleUICommand(intent);
            else
                await this.fail();
            intent = await this.nextIntent();
        }
        if (expected === ValueCategory.YesNo)
            return intent.value.value;
        else
            return intent.value;
    }
    async askChoices(question, choices) {
        await this.reply(question);
        this.setExpected(ValueCategory.MultipleChoice);
        this._choices = choices;
        for (let i = 0; i < choices.length; i++)
            await this.replyChoice(i, choices[i]);
        let intent = await this.nextIntent();
        while (!intent.isMultipleChoiceAnswer) {
            if (intent.isUICommand)
                await this._handleUICommand(intent);
            else
                await this.fail();
            intent = await this.nextIntent();
        }
        return intent.value;
    }

    async reset() {
        this.icon = null;
        this._dialogueState = null;
        await this.setExpected(null);
    }

    async replyInterp(msg, args, icon) {
        if (args === undefined)
            return this.reply(msg, icon);
        else
            return this.reply(this.interpolate(msg, args), icon);
    }

    async reply(msg, icon) {
        await this.conversation.sendReply(msg, icon || this.icon);
        return true;
    }

    async replyRDL(rdl, icon) {
        await this.conversation.sendRDL(rdl, icon || this.icon);
        return true;
    }

    async replyChoice(idx, title) {
        await this.conversation.sendChoice(idx, title);
        return true;
    }

    async replyButton(text, json) {
        await this.conversation.sendButton(text, json);
        return true;
    }

    async replySpecial(text, special) {
        let json = { code: ['bookkeeping', 'special', 'special:' + special], entities: {} };
        return this.replyButton(text, json);
    }

    async replyPicture(url, icon) {
        await this.conversation.sendPicture(url, icon || this.icon);
        return true;
    }

    async replyLink(title, url) {
        await this.conversation.sendLink(title, url);
        return true;
    }

    async replyResult(message, icon) {
        await this.conversation.sendResult(message, icon || this.icon);
        return true;
    }

    _isInDefaultState() {
        return this._notifyQueue.hasWaiter();
    }

    dispatchNotify(appId, icon, outputType, outputValue) {
        let item = new QueueItem.Notification(appId, icon, outputType, outputValue);
        this._pushQueueItem(item);
    }
    dispatchNotifyError(appId, icon, error) {
        let item = new QueueItem.Error(appId, icon, error);
        this._pushQueueItem(item);
    }

    start(showWelcome) {
        let promise = this._waitNextIntent();
        this._loop(showWelcome).then(() => {
            throw new Error('Unexpected end of dialog loop');
        }, (err) => {
            console.error('Uncaught error in dialog loop', err);
            throw err;
        });
        return promise;
    }

    _pushQueueItem(item) {
        // ensure that we have something to wait on before the next
        // command is handled
        if (!this._mgrPromise)
            this._waitNextIntent();

        this._notifyQueue.push(item);
    }

    _waitNextIntent() {
        let promise = new Promise((callback, errback) => {
            this._mgrResolve = callback;
        });
        this._mgrPromise = promise;
        return promise;
    }

    pushIntent(intent, confident = false) {
        this._pushQueueItem(new QueueItem.UserInput(intent, confident));
    }

    async handle(intent, confident=false) {
        // wait until the dialog is ready to accept commands
        await this._mgrPromise;
        assert(this._mgrPromise === null);

        const promise = this._waitNextIntent();

        if (this._isInDefaultState())
            this.pushIntent(intent, confident);
        else
            this._userInputQueue.push(intent);

        return promise;
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const assert = require('assert');	54✔
23
24	const ThingTalk = require('thingtalk');	54✔
25	const Ast = ThingTalk.Ast;	54✔
26	const interpolate = require('string-interp');	54✔
27	const AsyncQueue = require('consumer-queue');	54✔
28
29	const { getProgramIcon } = require('../utils/icons');	54✔
30	const ValueCategory = require('./value-category');	54✔
31	const QueueItem = require('./dialogue_queue');	54✔
32	const { CancellationError } = require('./errors');	54✔
33
34	const Helpers = require('./helpers');	54✔
35	const { showNotification, showError } = require('./notifications');	54✔
36	const { getFallbackExamples } = require('./fallback');	54✔
37	const { computeNewState, prepareContextForPrediction } = require('./dialogue_state_utils');	54✔
38	const DialoguePolicy = require('./dialogue_policy');	54✔
39
40	const ExecutionDialogueAgent = require('./execution_dialogue_agent');	54✔
41
42	// TODO: load the policy.yaml file instead
43	const POLICY_NAME = 'org.thingpedia.dialogue.transaction';	54✔
44	const TERMINAL_STATE = 'sys_end';	54✔
45
46	module.exports = class DialogueLoop {	54✔
47	constructor(conversation, engine, debug) {
48	this._userInputQueue = new AsyncQueue();	2✔
49	this._notifyQueue = new AsyncQueue();	2✔
50
51	this._debug = debug;	2✔
52	this.conversation = conversation;	2✔
53	this.engine = engine;	2✔
54	this._prefs = engine.platform.getSharedPreferences();	2✔
55	this.formatter = new ThingTalk.Formatter(engine.platform.locale, engine.platform.timezone, engine.schemas, conversation.gettext);	2✔
56	this.icon = null;	2✔
57	this.expecting = null;	2✔
58	this.platformData = null;	2✔
59	this._choices = null;	2✔
60
61	this._mgrResolve = null;	2✔
62	this._mgrPromise = null;	2✔
63
64	this._agent = new ExecutionDialogueAgent(engine, this, debug);	2✔
65	this._policy = new DialoguePolicy(this, conversation);	2✔
66	this._dialogueState = null; // thingtalk dialogue state	2✔
67	this._executorState = undefined; // private object managed by DialogueExecutor	2✔
68	this._lastNotificationApp = undefined;	2✔
69	}
70
71	get _() {
72	return this.conversation._;	3✔
73	}
74	get isAnonymous() {
75	return this.conversation.isAnonymous;	×
76	}
77
78	debug() {
79	if (!this._debug)	112!
80	return;	×
81	console.log.apply(console, arguments);	112✔
82	}
83
84	interpolate(msg, args) {
85	return interpolate(msg, args, {	1✔
86	locale: this.locale,
87	timezone: this.timezone
88	});
89	}
90
91	async nextIntent() {
92	await this.conversation.sendAskSpecial();	28✔
93	this._mgrPromise = null;	28✔
94	this._mgrResolve();	28✔
95	const intent = await this._userInputQueue.pop();	28✔
96	this.platformData = intent.platformData;	27✔
97	return intent;	27✔
98	}
99
100	_checkPolicy(policyName) {
101	if (policyName !== POLICY_NAME) {	57!
102	// TODO we should download the policy from Thingpedia
103	throw new Error(`Invalid dialogue policy ${policyName}`);	×
104	}
105	}
106
107	async _handleUICommand(intent) {
108	switch (intent.type) {	35!
109	case 'stop':
110	// stop means cancel, but without a failure message
111	throw new CancellationError();	34✔
112
113	case 'nevermind':
114	await this.reply(this._("Sorry I couldn't help on that."));	1✔
115	throw new CancellationError();	1✔
116
117	case 'debug':
118	await this.reply("Current State:\n" + this._dialogueState.prettyprint());	×
119	break;	×
120
121	case 'wakeup':
122	// nothing to do
123	break;	×
124
125	default:
126	await this.fail();	×
127	}
128	}
129
130	async _computePrediction(intent) {
131	// handle all intents generated internally and by the UI:
132	//
133	// - Failed when parsing fails
134	// - Answer when the user clicks a button, or when the agent is in "raw mode"
135	// - NeverMind when the user clicks the X button
136	// - Debug when the user clicks/types "debug"
137	// - WakeUp when the user says the wake word and nothing else
138	if (intent.isFailed) {	93✔
139	await getFallbackExamples(this, intent.utterance);	1✔
140	return null;	1✔
141	}
142	if (intent.isUnsupported) {	92!
NEW 143	this.icon = null;	×
144	await this.reply(this._("Sorry, I don't know how to do that yet."));	×
NEW 145	throw new CancellationError();	×
146	}
147	if (intent.isAnswer) {	92✔
148	const handled = await this._policy.handleAnswer(this._dialogueState, intent.value);	9✔
149	if (!handled) {	9!
150	await this.fail();	×
151	return null;	×
152	}
153	return handled;	9✔
154	}
155
156	if (intent.isProgram) {	83!
157	// convert thingtalk programs to dialogue states so we can use "\t" without too much typing
158	const prediction = new Ast.DialogueState(null, 'org.thingpedia.dialogue.transaction', 'execute', null, []);	×
159	for (let stmt of intent.program.rules)	×
160	prediction.history.push(new Ast.DialogueHistoryItem(null, stmt, null, 'accepted'));	×
161	return prediction;	×
162	}
163
164	if (intent.isUICommand) {	83✔
165	await this._handleUICommand(intent);	35✔
166	return null;	×
167	}
168
169	assert(intent.isDialogueState);	48✔
170	return intent.prediction;	48✔
171	}
172
173	_useNeuralNLG() {
174	return this._prefs.get('experimental-use-neural-nlg');	57✔
175	}
176
177	async _doAgentReply() {
178	let oldState = this._dialogueState;	57✔
179
180	let expect, utterance;
181	if (this._useNeuralNLG()) {	57!
182	[this._dialogueState, expect] = await this._policy.chooseAction(this._dialogueState);	×
183
184	const policyPrediction = computeNewState(oldState, this._dialogueState);	×
185	this.debug(`Agent act:`);	×
186	this.debug(policyPrediction.prettyprint());	×
187
188	const context = prepareContextForPrediction(oldState, 'agent');	×
189	await this.conversation.setContext(context, { allocateEntities: true, typeAnnotations: false });	×
190
191	utterance = await this.conversation.generateAnswer(policyPrediction);	×
192	} else {
193	[this._dialogueState, expect, utterance] = await this._policy.chooseAction(this._dialogueState);	57✔
194	}
195
196	this.icon = getProgramIcon(this._dialogueState);	57✔
197	await this.reply(utterance);	57✔
198	if (this._dialogueState.dialogueAct === TERMINAL_STATE)	57✔
199	throw new CancellationError();	5✔
200
201	await this.setExpected(expect);	52✔
202	return expect;	52✔
203	}
204
205	async _handleUserInput(intent) {
206	for (;;) {	66✔
207	const prediction = await this._computePrediction(intent);	93✔
208	if (prediction === null) {	58✔
209	intent = await this.nextIntent();	1✔
210	continue;	1✔
211	}
212	this._dialogueState = computeNewState(this._dialogueState, prediction);	57✔
213	this._checkPolicy(this._dialogueState.policy);	57✔
214	this.icon = getProgramIcon(this._dialogueState);	57✔
215
216	//this.debug(`Before execution:`);
217	//this.debug(this._dialogueState.prettyprint());
218
219	[this._dialogueState, this._executorState] = await this._agent.execute(this._dialogueState, this._executorState);	57✔
220	this.debug(`Execution state:`);	56✔
221	this.debug(this._dialogueState.prettyprint());	56✔
222
223	const expect = await this._doAgentReply();	56✔
224	if (expect === null)	51✔
225	return;	24✔
226
227	intent = await this.nextIntent();	27✔
228	}
229	}
230
231	async _handleAPICall(call) {
232	if (call.isNotification) {	×
233	await showNotification(this, call.appId, call.icon, call.outputType, call.outputValue, this._lastNotificationApp);	×
234	this._lastNotificationApp = call.appId;	×
235	} else if (call.isError) {	×
236	await showError(this, call.appId, call.icon, call.error, this._lastNotificationApp);	×
237	this._lastNotificationApp = call.appId;	×
238	}
239	}
240
241	async _loop(showWelcome) {
242	// if we want to show the welcome message, we run the policy on the `null` state, which will return the sys_greet intent
243	if (showWelcome) {	1!
244	await this._doAgentReply();	1✔
245	// reset the dialogue state here; if we don't, we we'll see sys_greet as an agent
246	// dialogue act; this is never seen in training, because in training the user speaks
247	// first, so it confuses the neural network
248	this._dialogueState = null;	1✔
249	// the utterance ends with "what can i do for you?", which is expect = 'generic'
250	// but we don't want to keep the microphone open here, we want to go back to wake-word mode
251	// so we unconditionally close the round here
252	await this.setExpected(null);	1✔
253	}
254
255	for (;;) {	1✔
256	const item = await this.nextQueueItem();	66✔
257	try {	66✔
258	if (item.isUserInput) {	66!
259	this._lastNotificationApp = undefined;	66✔
260	await this._handleUserInput(item.intent);	66✔
261	} else {
262	await this._handleAPICall(item);	×
263	this._dialogueState = null;	×
264	}
265	} catch(e) {
266	if (e.code === 'ECANCELLED') {	41!
267	await this.reset();	41✔
268	} else {
269	if (item.isUserInput) {	×
270	await this.replyInterp(this._("Sorry, I had an error processing your command: ${error}."), {//"	×
271	error: Helpers.formatError(this, e)
272	});
273	} else {
274	await this.replyInterp(this._("Sorry, that did not work: ${error}."), {	×
275	error: Helpers.formatError(this, e)
276	});
277	}
278	console.error(e);	×
279	}
280	}
281	}
282	}
283
284	get dialogueState() {
285	return this._dialogueState;	×
286	}
287
288	set dialogueState(newState) {
289	this._dialogueState = newState;	×
290	}
291
292	async nextQueueItem() {
293	this.setExpected(null);	66✔
294	await this.conversation.sendAskSpecial();	66✔
295	this._mgrPromise = null;	66✔
296	this._mgrResolve();	66✔
297	const queueItem = await this._notifyQueue.pop();	66✔
298	if (queueItem.isUserInput)	66!
299	this.platformData = queueItem.intent.platformData;	66✔
300	else
301	this.platformData = {};	×
302	return queueItem;	66✔
303	}
304
305	async lookingFor() {
306	const ALLOWED_UNITS = {	×
307	'ms': ['ms', 's', 'min', 'h', 'day', 'week', 'mon', 'year'],
308	'm': ['m', 'km', 'mm', 'cm', 'mi', 'in'],
309	'mps': ['mps', 'kmph', 'mph'],
310	'kg': ['kg', 'g', 'lb', 'oz'],
311	'Pa': ['Pa', 'bar', 'psi', 'mmHg', 'inHg', 'atm'],
312	'C': ['C', 'F', 'K'],
313	'kcal': ['kcal', 'kJ'],
314	'byte': ['byte', 'KB', 'KiB', 'MB', 'MiB', 'GB', 'GiB', 'TB', 'TiB']
315	};
316
317	if (this.expecting === null) {	×
318	await this.reply(this._("In fact, I did not ask for anything at all!"));	×
319	} else if (this.expecting === ValueCategory.YesNo) {	×
320	await this.reply(this._("Sorry, I need you to confirm the last question first."));	×
321	} else if (this.expecting === ValueCategory.MultipleChoice) {	×
322	await this.reply(this._("Could you choose one of the following?"));	×
323	this.conversation.resendChoices();	×
324	} else if (this.expecting.isMeasure) {	×
325	await this.replyInterp(this._("I'm looking for ${expecting:select:\	×
326	ms {a time interval}\
327	m {a length}\
328	mps {a speed}\
329	kg {a weight}\
330	Pa {a pressure}\
331	C {a temperature}\
332	kcal {an energy}\
333	byte {a size}\
334	other {a value}\
335	} in any of the supported units (${units})."), {
336	expecting: this.expecting.unit,
337	units: ALLOWED_UNITS[this.expecting.unit]
338	});
339	} else if (this.expecting === ValueCategory.Number) {	×
340	await this.reply(this._("Could you give me a number?"));	×
341	} else if (this.expecting === ValueCategory.Date) {	×
342	await this.reply(this._("Could you give me a date?"));	×
343	} else if (this.expecting === ValueCategory.Time) {	×
344	await this.reply(this._("Could you give me a time of day?"));	×
345	} else if (this.expecting === ValueCategory.Picture) {	×
346	await this.reply(this._("Could you upload a picture?"));	×
347	} else if (this.expecting === ValueCategory.Location) {	×
348	await this.reply(this._("Could you give me a place?"));	×
349	} else if (this.expecting === ValueCategory.PhoneNumber) {	×
350	await this.reply(this._("Could you give me a phone number?"));	×
351	} else if (this.expecting === ValueCategory.EmailAddress) {	×
352	await this.reply(this._("Could you give me an email address?"));	×
353	} else if (this.expecting === ValueCategory.RawString \|\| this.expecting === ValueCategory.Password) {	×
354	// ValueCategory.RawString puts Almond in raw mode,
355	// so we accept almost everything
356	// but this will happen if the user clicks a button
357	// or upload a picture
358	await this.reply(this._("Which is interesting, because I'll take anything at all. Just type your mind!"));	×
359	} else if (this.expecting === ValueCategory.Command) {	×
360	await this.reply(this._("I'm looking for a command."));	×
361	} else if (this.expecting === ValueCategory.Predicate) {	×
362	await this.reply(this._("I'm looking for a filter"));	×
363	} else {
364	await this.reply(this._("In fact, I'm not even sure what I asked. Sorry!"));	×
365	}
366	}
367
368	async fail(msg) {
369	if (this.expecting === null) {	×
370	if (msg) {	×
371	await this.replyInterp(this._("Sorry, I did not understand that: ${error}. Can you rephrase it?"), {	×
372	error: msg
373	});
374	} else {
375	await this.reply(this._("Sorry, I did not understand that. Can you rephrase it?"));	×
376	}
377	} else {
378	if (msg)	×
379	await this.replyInterp(this._("Sorry, I did not understand that: ${error}."), { error: msg });	×
380	else
381	await this.reply(this._("Sorry, I did not understand that."));	×
382	await this.lookingFor();	×
383	}
384	return true;	×
385	}
386
387	setExpected(expected) {
388	if (expected === undefined)	161!
389	throw new TypeError();	×
390	this.expecting = expected;	161✔
391	const context = prepareContextForPrediction(this._dialogueState, 'user');	161✔
392	this.conversation.setContext(context, { typeAnnotations: false });	161✔
393	this.conversation.expect(expected);	161✔
394	}
395
396	/**
397	* Ask a question to the user.
398	*
399	* This is a legacy method used for certain scripted interactions.
400	*/
401	async ask(expected, question, args) {
402	await this.replyInterp(question, args);	×
403	await this.setExpected(expected);	×
404	let intent = await this.nextIntent();	×
405	while (!intent.isAnswer \|\| intent.category !== expected) {	×
406	if (intent.isUICommand)	×
407	await this._handleUICommand(intent);	×
408	else
409	await this.fail();	×
410	intent = await this.nextIntent();	×
411	}
412	if (expected === ValueCategory.YesNo)	×
413	return intent.value.value;	×
414	else
415	return intent.value;	×
416	}
417	async askChoices(question, choices) {
418	await this.reply(question);	×
419	this.setExpected(ValueCategory.MultipleChoice);	×
420	this._choices = choices;	×
421	for (let i = 0; i < choices.length; i++)	×
422	await this.replyChoice(i, choices[i]);	×
423	let intent = await this.nextIntent();	×
424	while (!intent.isMultipleChoiceAnswer) {	×
425	if (intent.isUICommand)	×
426	await this._handleUICommand(intent);	×
427	else
428	await this.fail();	×
429	intent = await this.nextIntent();	×
430	}
431	return intent.value;	×
432	}
433
434	async reset() {
435	this.icon = null;	42✔
436	this._dialogueState = null;	42✔
437	await this.setExpected(null);	42✔
438	}
439
440	async replyInterp(msg, args, icon) {
441	if (args === undefined)	1!
442	return this.reply(msg, icon);	×
443	else
444	return this.reply(this.interpolate(msg, args), icon);	1✔
445	}
446
447	async reply(msg, icon) {
448	await this.conversation.sendReply(msg, icon \|\| this.icon);	60✔
449	return true;	60✔
450	}
451
452	async replyRDL(rdl, icon) {
453	await this.conversation.sendRDL(rdl, icon \|\| this.icon);	×
454	return true;	×
455	}
456
457	async replyChoice(idx, title) {
458	await this.conversation.sendChoice(idx, title);	×
459	return true;	×
460	}
461
462	async replyButton(text, json) {
463	await this.conversation.sendButton(text, json);	×
464	return true;	×
465	}
466
467	async replySpecial(text, special) {
468	let json = { code: ['bookkeeping', 'special', 'special:' + special], entities: {} };	×
469	return this.replyButton(text, json);	×
470	}
471
472	async replyPicture(url, icon) {
473	await this.conversation.sendPicture(url, icon \|\| this.icon);	×
474	return true;	×
475	}
476
477	async replyLink(title, url) {
478	await this.conversation.sendLink(title, url);	×
479	return true;	×
480	}
481
482	async replyResult(message, icon) {
483	await this.conversation.sendResult(message, icon \|\| this.icon);	×
484	return true;	×
485	}
486
487	_isInDefaultState() {
488	return this._notifyQueue.hasWaiter();	93✔
489	}
490
491	dispatchNotify(appId, icon, outputType, outputValue) {
492	let item = new QueueItem.Notification(appId, icon, outputType, outputValue);	25✔
493	this._pushQueueItem(item);	25✔
494	}
495	dispatchNotifyError(appId, icon, error) {
496	let item = new QueueItem.Error(appId, icon, error);	3✔
497	this._pushQueueItem(item);	3✔
498	}
499
500	start(showWelcome) {
501	let promise = this._waitNextIntent();	1✔
502	this._loop(showWelcome).then(() => {	1✔
503	throw new Error('Unexpected end of dialog loop');	×
504	}, (err) => {
505	console.error('Uncaught error in dialog loop', err);	×
506	throw err;	×
507	});
508	return promise;	1✔
509	}
510
511	_pushQueueItem(item) {
512	// ensure that we have something to wait on before the next
513	// command is handled
514	if (!this._mgrPromise)	94✔
515	this._waitNextIntent();	1✔
516
517	this._notifyQueue.push(item);	94✔
518	}
519
520	_waitNextIntent() {
521	let promise = new Promise((callback, errback) => {	95✔
522	this._mgrResolve = callback;	95✔
523	});
524	this._mgrPromise = promise;	95✔
525	return promise;	95✔
526	}
527
528	pushIntent(intent, confident = false) {	×
529	this._pushQueueItem(new QueueItem.UserInput(intent, confident));	66✔
530	}
531
532	async handle(intent, confident=false) {	1✔
533	// wait until the dialog is ready to accept commands
534	await this._mgrPromise;	93✔
535	assert(this._mgrPromise === null);	93✔
536
537	const promise = this._waitNextIntent();	93✔
538
539	if (this._isInDefaultState())	93✔
540	this.pushIntent(intent, confident);	66✔
541	else
542	this._userInputQueue.push(intent);	27✔
543
544	return promise;	93✔
545	}
546	};

stanford-oval / genie-toolkit / 1310

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous