2369

pending completion

Build # 2369

Build Type

Pull #787

travis-ci-com

Committed by

web-flow

Commit Message

Merge ba964406e into 577e0100b

Pull Request Pull Request #787: Add a new audio control protocol

Run Details

10143 of 20495 branches covered (49.49%)

Branch coverage included in aggregate %.

98 of 280 new or added lines in 17 files covered. (35.0%)

6 existing lines in 4 files now uncovered.

24230 of 40585 relevant lines covered (59.7%)

374911.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.0

/lib/dialogue-agent/conversation.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as events from 'events';
import * as ThingTalk from 'thingtalk';
import * as Stream from "stream";

import * as I18n from '../i18n';

import ValueCategory from './value-category';
import { DialogueLoop } from './dialogue-loop';
import { PlatformData, MessageType, Message, RDL } from './protocol';
import { EntityMap } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';
import type Engine from '../engine';
import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";
import AppExecutor from '../engine/apps/app_executor';
import ConversationLogger from './logging';
import { ConversationStateRow, LocalTable } from "../engine/db";

const DummyStatistics = {
    hit() {
    }
};

const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes

export interface ConversationOptions {
    nluServerUrl ?: string;
    nlgServerUrl ?: string;
    anonymous ?: boolean;
    debug ?: boolean;
    rng ?: () => number;
    inactivityTimeout ?: number;
    contextResetTimeout ?: number;
    showWelcome ?: boolean;
    deleteWhenInactive ?: boolean;
    log ?: boolean;
    dialogueFlags ?: Record<string, boolean>;
    useConfidence ?: boolean;
    faqModels ?: Record<string, {
        url : string;
        highConfidence ?: number;
        lowConfidence ?: number;
    }>;
    syncDevices ?: boolean;
}

interface Statistics {
    hit(key : string) : void;
}

export interface ConversationDelegate {
    setHypothesis(hyp : string) : Promise<void>;
    setExpected(expect : string|null, ctx : {
        code : string[];
        entities : EntityMap;
    }) : Promise<void>;
    addMessage(msg : Message) : Promise<void>;
}

export interface ConversationState {
    history : Message[];
    dialogueState : Record<string, unknown>;
    lastMessageId : number;
}

/**
 * A single session of conversation in Genie.
 *
 * This object is responsible for maintaining the history of the conversation
 * to support clients reconnecting to the same conversation later, as well
 * as tracking connected clients and inactivity timeouts.
 */
export default class Conversation extends events.EventEmitter {
    // NOTE: The actual conversation logic is in DialogueLoop.
    private _engine : Engine;
    private _conversationId : string;
    private _locale : string;
    _ : (x : string) => string;

    private _stats : Statistics;
    private _options : ConversationOptions;
    private _debug : boolean;
    private _dialogueFlags : Record<string, boolean>;
    rng : () => number;

    private _loop : DialogueLoop;
    private _expecting : ValueCategory|null;
    private _context : {
        code : string[];
        entities : EntityMap;
    };

    private _started : boolean;
    private _delegates : Set<ConversationDelegate>;
    private _history : Message[];
    private _nextMsgId : number;
    private _inactivityTimeout : NodeJS.Timeout|null;
    private _inactivityTimeoutSec : number;
    private _contextResetTimeout : NodeJS.Timeout|null;
    private _contextResetTimeoutSec : number;

    private _log : ConversationLogger;
    private readonly _conversationStateDB : LocalTable<ConversationStateRow>;

    constructor(engine : Engine,
                conversationId : string,
                options : ConversationOptions = {}) {
        super();
        this._engine = engine;
        this._conversationStateDB = this._engine.db.getLocalTable('conversation_state');

        this._conversationId = conversationId;
        this._locale = this._engine.platform.locale;
        this._ = I18n.get(this._locale).gettext;

        const stats = this._engine.platform.getCapability('statistics');
        if (stats === null)
            this._stats = DummyStatistics;
        else
            this._stats = stats;

        this._options = options;
        this._debug = !!this._options.debug;
        this._dialogueFlags = options.dialogueFlags || {};

        this.rng = options.rng || Math.random;

        this._loop = new DialogueLoop(this, this._engine, {
            nluServerUrl: options.nluServerUrl,
            nlgServerUrl: options.nlgServerUrl,
            faqModels: options.faqModels || {},
            useConfidence: options.useConfidence ?? true,
            debug: this._debug,
            rng: this.rng,
        });
        this._expecting = null;
        this._context = { code: ['null'], entities: {} };
        this._delegates = new Set;
        this._history = [];
        this._nextMsgId = 0;
        this._started = false;

        this._inactivityTimeout = null;
        this._inactivityTimeoutSec = options.inactivityTimeout || DEFAULT_CONVERSATION_TTL;
        this._contextResetTimeout = null;
        this._contextResetTimeoutSec = options.contextResetTimeout || this._inactivityTimeoutSec;

        this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);
    }

    get isAnonymous() : boolean {
        return !!this._options.anonymous;
    }

    get id() : string {
        return this._conversationId;
    }

    get engine() : Engine {
        return this._engine;
    }

    get stats() : Statistics {
        return this._stats;
    }

    get history() : Message[] {
        return this._history;
    }

    get inRecordingMode() : boolean {
        return !!this._options.log;
    }

    get dialogueFlags() : Record<string, boolean> {
        return this._dialogueFlags;
    }

    startRecording() {
        this._options.log = true;
    }

    async endRecording() {
        await this._log.dialogueFinished();
        this._options.log = false;
    }

    notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
        return this._loop.dispatchNotify(app, outputType, outputValue);
    }

    notifyError(app : AppExecutor, error : Error) {
        return this._loop.dispatchNotifyError(app, error);
    }

    setExpected(expecting : ValueCategory|null, context : {
        code : string[];
        entities : EntityMap;
    }) : void {
        this._expecting = expecting;
        this._context = context;
    }

    async start(state ?: ConversationState) : Promise<void> {
        this._resetInactivityTimeout();

        if (state) {
            for (const msg of state.history)
                await this.addMessage(msg);
            this._nextMsgId = state.lastMessageId+1;
        }
        this._started = true;

        return this._loop.start(!!this._options.showWelcome,
            state ? state.dialogueState : null);
    }

    async stop() : Promise<void> {
        return this._loop.stop();
    }

    private _resetInactivityTimeout() {
        // after "options.inactivityTimeout" milliseconds we inform the app that the conversation
        // is inactive (turn off LEDs, close the microphone, etc.)
        if (this._inactivityTimeout)
            clearTimeout(this._inactivityTimeout);
        if (this._inactivityTimeoutSec > 0) {
            this._inactivityTimeout = setTimeout(() => {
                this.emit('inactive');
            }, this._inactivityTimeoutSec);
        }

        // after "options.contextResetTimeout", we reset the context, forgetting the state of the
        // conversation
        if (this._contextResetTimeout)
            clearTimeout(this._contextResetTimeout);
        if (this._contextResetTimeoutSec > 0) {
            this._contextResetTimeout = setTimeout(() => {
                this._loop.reset();
            }, this._contextResetTimeoutSec);
        }
    }

    async addOutput(out : ConversationDelegate, replayHistory = true) {
        this._delegates.add(out);
        if (replayHistory) {
            for (const msg of this._history) {
                if (!await this._callDelegate(out, (out) => out.addMessage(msg)))
                    return;
            }
        }

        if (this._started) {
            const what = ValueCategory.toString(this._expecting);
            await this._callDelegate(out, (out) => out.setExpected(what, this._context));
        }
    }
    async removeOutput(out : ConversationDelegate) {
        this._delegates.delete(out);
    }

    private async _callDelegate(out : ConversationDelegate, fn : (out : ConversationDelegate) => unknown) {
        try {
            await fn(out);
            return true;
        } catch(e) {
            // delegate disappeared (likely a disconnected websocket)
            this._delegates.delete(out);
            return false;
        }
    }

    private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
        return Promise.all(Array.from(this._delegates).map((out) => this._callDelegate(out, fn)));
    }

    async setHypothesis(hypothesis : string) : Promise<void> {
        await this._callDelegates((out) => out.setHypothesis(hypothesis));
    }

    async sendAskSpecial() : Promise<void> {
        const what = ValueCategory.toString(this._expecting);

        if (this._debug) {
            if (what !== null && what !== 'generic')
                console.log('Genie sends a special request');
            else if (what !== null)
                console.log('Genie expects an answer');
        }

        await this._callDelegates((out) => out.setExpected(what, this._context));
    }

    /**
     * Add a message to the conversation history.
     *
     * This method is exported to inject conversation history from outside.
     */
    async addMessage(msg : Message) {
        if (msg.id !== undefined)
            this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);
        else
            msg.id = this._nextMsgId ++;
        this._history.push(msg);
        if (this._history.length > 30)
            this._history.shift();
        await this._callDelegates((out) => out.addMessage(msg));

        await this.saveState(msg.id);
    }

    async saveState(lastMessageId : number) {
        const conversationState = this.getState();
        const row = {
            history: JSON.stringify(/* FIXME conversationState.history */ []),
            dialogueState: JSON.stringify(conversationState.dialogueState),
            lastMessageId: lastMessageId,
        };
        await this._conversationStateDB.insertOne(this._conversationId, row);
    }

    /**
     * Extract the state from the conversation.
     *
     * This method is provided to save and restore the conversation state,
     * and transfer the conversation state between engines.
     */
    getState() : ConversationState {
        const history = this._history.slice();
        const lastMessageId = history.length > 0 ? history[history.length-1].id : null;
        return {
            history: history,
            dialogueState: this._loop.getState(),
            lastMessageId: lastMessageId ? lastMessageId : 0
        };
    }

    async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        // if the command is just whitespace, ignore it without even adding it to the history
        if (!command.trim())
            return;

        this.stats.hit('sabrina-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received assistant command ' + command);

        await this._loop.handleCommand({ type: 'command', utterance: command, platformData });
    }

    async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;
        this.stats.hit('sabrina-parsed-command');
        this.emit('active');
        this._resetInactivityTimeout();
        if (typeof root === 'string')
            root = JSON.parse(root);
        await this.addMessage({ type: MessageType.COMMAND, command: title || command, json: root });

        if (this._debug)
            console.log('Received pre-parsed assistant command');
        if (root.example_id) {
            this._engine.thingpedia.clickExample(root.example_id).catch((e) => {
                console.error('Failed to record example click: ' + e.message);
            });
        }

        if ('program' in root)
            return this.handleThingTalk(root.program, platformData);

        const { code, entities } = root;
        for (const name in entities) {
            if (name.startsWith('SLOT_')) {
                const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];
                const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);
                const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);
                entities[name] = value;
            }
        }

        const parsed = await ThingTalkUtils.parsePrediction(code, entities, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        }, true);
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\t ${program}`;
        this.stats.hit('sabrina-thingtalk-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received ThingTalk program');

        const parsed = await ThingTalkUtils.parse(program, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        });
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    sendReply(message : string, icon : string|null) {
        if (this._debug)
            console.log('Genie says: ' + message);
        return this.addMessage({ type: MessageType.TEXT, text: message, icon });
    }

    sendMedia(mediaType : 'picture'|'audio'|'video', url : string, alt : string|undefined, icon : string|null) {
        if (this._debug)
            console.log('Genie sends ' + mediaType + ': '+ url);
        return this.addMessage({ type: mediaType as MessageType.AUDIO|MessageType.VIDEO|MessageType.PICTURE, url, alt, icon });
    }

    sendRDL(rdl : RDL, icon : string|null) {
        if (this._debug)
            console.log('Genie sends RDL: '+ rdl.callback);
        return this.addMessage({ type: MessageType.RDL, rdl, icon });
    }

    sendSoundEffect(name : string, exclusive = false, icon : string|null) {
        if (this._debug)
            console.log('Genie sends sound effect: '+ name);
        return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });
    }

    sendChoice(idx : number, title : string) {
        if (this._expecting !== ValueCategory.MultipleChoice)
            console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');
        if (this._debug)
            console.log('Genie sends multiple choice button: '+ title);
        return this.addMessage({ type: MessageType.CHOICE, idx, title });
    }

    sendButton(title : string, json : string) {
        if (this._debug)
            console.log('Genie sends generic button: '+ title);
        return this.addMessage({ type: MessageType.BUTTON, json, title });
    }

    sendLink(title : string, url : string, state : ConversationState) {
        if (this._debug)
            console.log('Genie sends link: '+ url);
        return this.addMessage({ type: MessageType.LINK, url, title, state });
    }

    sendNewProgram(program : {
        uniqueId : string;
        name : string;
        code : string;
        results : Array<Record<string, unknown>>;
        errors : string[];
        icon : string|null;
    }) {
        if (this._debug)
            console.log('Genie executed new program: '+ program.uniqueId);
        return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });
    }

    async dialogueFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.dialogueFinished();
    }

    async turnFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.turnFinished();
    }

    async voteLast(vote : 'up'|'down') {
        if (!this.inRecordingMode)
            return;
        await this._log.voteLast(vote);
    }

    async commentLast(comment : string) {
        if (!this.inRecordingMode)
            return;
        await this._log.commentLast(comment);
    }

    updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'|'user_timestamp'>, value : string) {
        if (!this.inRecordingMode)
            return;
        this._log.updateLog(field, value);
    }

    readLog() {
        const readable = Stream.Readable.from(this._log.read());
        const serializer = new DialogueSerializer({ annotations: true });
        return readable.pipe(serializer);
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as events from 'events';	97✔
22	import * as ThingTalk from 'thingtalk';	97✔
23	import * as Stream from "stream";	97✔
24
25	import * as I18n from '../i18n';	97✔
26
27	import ValueCategory from './value-category';	97✔
28	import { DialogueLoop } from './dialogue-loop';	97✔
29	import { PlatformData, MessageType, Message, RDL } from './protocol';	97✔
30	import { EntityMap } from '../utils/entity-utils';
31	import * as ThingTalkUtils from '../utils/thingtalk';	97✔
32	import type Engine from '../engine';
33	import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";	97✔
34	import AppExecutor from '../engine/apps/app_executor';
35	import ConversationLogger from './logging';	97✔
36	import { ConversationStateRow, LocalTable } from "../engine/db";
37
38	const DummyStatistics = {	97✔
39	hit() {
40	}
41	};
42
43	const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes	97✔
44
45	export interface ConversationOptions {
46	nluServerUrl ?: string;
47	nlgServerUrl ?: string;
48	anonymous ?: boolean;
49	debug ?: boolean;
50	rng ?: () => number;
51	inactivityTimeout ?: number;
52	contextResetTimeout ?: number;
53	showWelcome ?: boolean;
54	deleteWhenInactive ?: boolean;
55	log ?: boolean;
56	dialogueFlags ?: Record<string, boolean>;
57	useConfidence ?: boolean;
58	faqModels ?: Record<string, {
59	url : string;
60	highConfidence ?: number;
61	lowConfidence ?: number;
62	}>;
63	syncDevices ?: boolean;
64	}
65
66	interface Statistics {
67	hit(key : string) : void;
68	}
69
70	export interface ConversationDelegate {
71	setHypothesis(hyp : string) : Promise<void>;
72	setExpected(expect : string\|null, ctx : {
73	code : string[];
74	entities : EntityMap;
75	}) : Promise<void>;
76	addMessage(msg : Message) : Promise<void>;
77	}
78
79	export interface ConversationState {
80	history : Message[];
81	dialogueState : Record<string, unknown>;
82	lastMessageId : number;
83	}
84
85	/**
86	* A single session of conversation in Genie.
87	*
88	* This object is responsible for maintaining the history of the conversation
89	* to support clients reconnecting to the same conversation later, as well
90	* as tracking connected clients and inactivity timeouts.
91	*/
92	export default class Conversation extends events.EventEmitter {	97✔
93	// NOTE: The actual conversation logic is in DialogueLoop.
94	private _engine : Engine;
95	private _conversationId : string;
96	private _locale : string;
97	_ : (x : string) => string;
98
99	private _stats : Statistics;
100	private _options : ConversationOptions;
101	private _debug : boolean;
102	private _dialogueFlags : Record<string, boolean>;
103	rng : () => number;
104
105	private _loop : DialogueLoop;
106	private _expecting : ValueCategory\|null;
107	private _context : {
108	code : string[];
109	entities : EntityMap;
110	};
111
112	private _started : boolean;
113	private _delegates : Set<ConversationDelegate>;
114	private _history : Message[];
115	private _nextMsgId : number;
116	private _inactivityTimeout : NodeJS.Timeout\|null;
117	private _inactivityTimeoutSec : number;
118	private _contextResetTimeout : NodeJS.Timeout\|null;
119	private _contextResetTimeoutSec : number;
120
121	private _log : ConversationLogger;
122	private readonly _conversationStateDB : LocalTable<ConversationStateRow>;
123
124	constructor(engine : Engine,
125	conversationId : string,
126	options : ConversationOptions = {}) {	×
127	super();	4✔
128	this._engine = engine;	4✔
129	this._conversationStateDB = this._engine.db.getLocalTable('conversation_state');	4✔
130
131	this._conversationId = conversationId;	4✔
132	this._locale = this._engine.platform.locale;	4✔
133	this._ = I18n.get(this._locale).gettext;	4✔
134
135	const stats = this._engine.platform.getCapability('statistics');	4✔
136	if (stats === null)	4!
137	this._stats = DummyStatistics;	4✔
138	else
139	this._stats = stats;	×
140
141	this._options = options;	4✔
142	this._debug = !!this._options.debug;	4✔
143	this._dialogueFlags = options.dialogueFlags \|\| {};	4✔
144
145	this.rng = options.rng \|\| Math.random;	4✔
146
147	this._loop = new DialogueLoop(this, this._engine, {	4✔
148	nluServerUrl: options.nluServerUrl,
149	nlgServerUrl: options.nlgServerUrl,
150	faqModels: options.faqModels \|\| {},	7✔
151	useConfidence: options.useConfidence ?? true,	12✔
152	debug: this._debug,
153	rng: this.rng,
154	});
155	this._expecting = null;	4✔
156	this._context = { code: ['null'], entities: {} };	4✔
157	this._delegates = new Set;	4✔
158	this._history = [];	4✔
159	this._nextMsgId = 0;	4✔
160	this._started = false;	4✔
161
162	this._inactivityTimeout = null;	4✔
163	this._inactivityTimeoutSec = options.inactivityTimeout \|\| DEFAULT_CONVERSATION_TTL;	4✔
164	this._contextResetTimeout = null;	4✔
165	this._contextResetTimeoutSec = options.contextResetTimeout \|\| this._inactivityTimeoutSec;	4✔
166
167	this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);	4✔
168	}
169
170	get isAnonymous() : boolean {
171	return !!this._options.anonymous;	113✔
172	}
173
174	get id() : string {
175	return this._conversationId;	90✔
176	}
177
178	get engine() : Engine {
179	return this._engine;	1✔
180	}
181
182	get stats() : Statistics {
183	return this._stats;	224✔
184	}
185
186	get history() : Message[] {
187	return this._history;	×
188	}
189
190	get inRecordingMode() : boolean {
191	return !!this._options.log;	1,255✔
192	}
193
194	get dialogueFlags() : Record<string, boolean> {
195	return this._dialogueFlags;	90✔
196	}
197
198	startRecording() {
199	this._options.log = true;	1✔
200	}
201
202	async endRecording() {
203	await this._log.dialogueFinished();	1✔
204	this._options.log = false;	1✔
205	}
206
207	notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
208	return this._loop.dispatchNotify(app, outputType, outputValue);	25✔
209	}
210
211	notifyError(app : AppExecutor, error : Error) {
212	return this._loop.dispatchNotifyError(app, error);	3✔
213	}
214
215	setExpected(expecting : ValueCategory\|null, context : {
216	code : string[];
217	entities : EntityMap;
218	}) : void {
219	this._expecting = expecting;	228✔
220	this._context = context;	228✔
221	}
222
223	async start(state ?: ConversationState) : Promise<void> {
224	this._resetInactivityTimeout();	3✔
225
226	if (state) {	3!
UNCOV 227	for (const msg of state.history)	×
228	await this.addMessage(msg);	×
229	this._nextMsgId = state.lastMessageId+1;	×
230	}
231	this._started = true;	3✔
232
233	return this._loop.start(!!this._options.showWelcome,	3✔
234	state ? state.dialogueState : null);	3!
235	}
236
237	async stop() : Promise<void> {
238	return this._loop.stop();	×
239	}
240
241	private _resetInactivityTimeout() {
242	// after "options.inactivityTimeout" milliseconds we inform the app that the conversation
243	// is inactive (turn off LEDs, close the microphone, etc.)
244	if (this._inactivityTimeout)	220✔
245	clearTimeout(this._inactivityTimeout);	217✔
246	if (this._inactivityTimeoutSec > 0) {	220!
247	this._inactivityTimeout = setTimeout(() => {	220✔
248	this.emit('inactive');	×
249	}, this._inactivityTimeoutSec);
250	}
251
252	// after "options.contextResetTimeout", we reset the context, forgetting the state of the
253	// conversation
254	if (this._contextResetTimeout)	220✔
255	clearTimeout(this._contextResetTimeout);	217✔
256	if (this._contextResetTimeoutSec > 0) {	220!
257	this._contextResetTimeout = setTimeout(() => {	220✔
258	this._loop.reset();	×
259	}, this._contextResetTimeoutSec);
260	}
261	}
262
263	async addOutput(out : ConversationDelegate, replayHistory = true) {	3✔
264	this._delegates.add(out);	3✔
265	if (replayHistory) {	3!
266	for (const msg of this._history) {	3✔
267	if (!await this._callDelegate(out, (out) => out.addMessage(msg)))	×
268	return;	×
269	}
270	}
271
272	if (this._started) {	3!
273	const what = ValueCategory.toString(this._expecting);	3✔
274	await this._callDelegate(out, (out) => out.setExpected(what, this._context));	3✔
275	}
276	}
277	async removeOutput(out : ConversationDelegate) {
278	this._delegates.delete(out);	×
279	}
280
281	private async _callDelegate(out : ConversationDelegate, fn : (out : ConversationDelegate) => unknown) {
282	try {	702✔
283	await fn(out);	702✔
284	return true;	702✔
285	} catch(e) {
286	// delegate disappeared (likely a disconnected websocket)
287	this._delegates.delete(out);	×
288	return false;	×
289	}
290	}
291
292	private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
293	return Promise.all(Array.from(this._delegates).map((out) => this._callDelegate(out, fn)));	704✔
294	}
295
296	async setHypothesis(hypothesis : string) : Promise<void> {
297	await this._callDelegates((out) => out.setHypothesis(hypothesis));	×
298	}
299
300	async sendAskSpecial() : Promise<void> {
301	const what = ValueCategory.toString(this._expecting);	220✔
302
303	if (this._debug) {	220!
304	if (what !== null && what !== 'generic')	220✔
305	console.log('Genie sends a special request');	22✔
306	else if (what !== null)	198✔
307	console.log('Genie expects an answer');	4✔
308	}
309
310	await this._callDelegates((out) => out.setExpected(what, this._context));	220✔
311	}
312
313	/**
314	* Add a message to the conversation history.
315	*
316	* This method is exported to inject conversation history from outside.
317	*/
318	async addMessage(msg : Message) {
319	if (msg.id !== undefined)	484!
320	this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);	×
321	else
322	msg.id = this._nextMsgId ++;	484✔
323	this._history.push(msg);	484✔
324	if (this._history.length > 30)	484✔
325	this._history.shift();	445✔
326	await this._callDelegates((out) => out.addMessage(msg));	484✔
327
328	await this.saveState(msg.id);	484✔
329	}
330
331	async saveState(lastMessageId : number) {
332	const conversationState = this.getState();	484✔
333	const row = {	484✔
334	history: JSON.stringify(/* FIXME conversationState.history */ []),
335	dialogueState: JSON.stringify(conversationState.dialogueState),
336	lastMessageId: lastMessageId,
337	};
338	await this._conversationStateDB.insertOne(this._conversationId, row);	484✔
339	}
340
341	/**
342	* Extract the state from the conversation.
343	*
344	* This method is provided to save and restore the conversation state,
345	* and transfer the conversation state between engines.
346	*/
347	getState() : ConversationState {
348	const history = this._history.slice();	491✔
349	const lastMessageId = history.length > 0 ? history[history.length-1].id : null;	491✔
350	return {	491✔
351	history: history,
352	dialogueState: this._loop.getState(),
353	lastMessageId: lastMessageId ? lastMessageId : 0	491✔
354	};
355	}
356
357	async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {	10✔
358	this._engine.updateActivity();	10✔
359	// if the command is just whitespace, ignore it without even adding it to the history
360	if (!command.trim())	10!
361	return;	×
362
363	this.stats.hit('sabrina-command');	10✔
364	this.emit('active');	10✔
365	this._resetInactivityTimeout();	10✔
366	await this.addMessage({ type: MessageType.COMMAND, command });	10✔
367	if (this._debug)	10!
368	console.log('Received assistant command ' + command);	10✔
369
370	await this._loop.handleCommand({ type: 'command', utterance: command, platformData });	10✔
371	}
372
373	async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {	92✔
374	this._engine.updateActivity();	92✔
375	const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;	92!
376	this.stats.hit('sabrina-parsed-command');	92✔
377	this.emit('active');	92✔
378	this._resetInactivityTimeout();	92✔
379	if (typeof root === 'string')	92!
380	root = JSON.parse(root);	×
381	await this.addMessage({ type: MessageType.COMMAND, command: title \|\| command, json: root });	92✔
382
383	if (this._debug)	92!
384	console.log('Received pre-parsed assistant command');	92✔
385	if (root.example_id) {	92!
386	this._engine.thingpedia.clickExample(root.example_id).catch((e) => {	×
387	console.error('Failed to record example click: ' + e.message);	×
388	});
389	}
390
391	if ('program' in root)	92!
392	return this.handleThingTalk(root.program, platformData);	×
393
394	const { code, entities } = root;	92✔
395	for (const name in entities) {	92✔
396	if (name.startsWith('SLOT_')) {	×
397	const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];	×
398	const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);	×
399	const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);	×
400	entities[name] = value;	×
401	}
402	}
403
404	const parsed = await ThingTalkUtils.parsePrediction(code, entities, {	92✔
405	thingpediaClient: this._engine.thingpedia,
406	schemaRetriever: this._engine.schemas,
407	loadMetadata: true
408	}, true);
409	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	92✔
410	}
411
412	async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {	115✔
413	this._engine.updateActivity();	115✔
414	const command = `\\t ${program}`;	115✔
415	this.stats.hit('sabrina-thingtalk-command');	115✔
416	this.emit('active');	115✔
417	this._resetInactivityTimeout();	115✔
418	await this.addMessage({ type: MessageType.COMMAND, command });	115✔
419	if (this._debug)	115!
420	console.log('Received ThingTalk program');	115✔
421
422	const parsed = await ThingTalkUtils.parse(program, {	115✔
423	thingpediaClient: this._engine.thingpedia,
424	schemaRetriever: this._engine.schemas,
425	loadMetadata: true
426	});
427	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	115✔
428	}
429
430	sendReply(message : string, icon : string\|null) {
431	if (this._debug)	133!
432	console.log('Genie says: ' + message);	133✔
433	return this.addMessage({ type: MessageType.TEXT, text: message, icon });	133✔
434	}
435
436	sendMedia(mediaType : 'picture'\|'audio'\|'video', url : string, alt : string\|undefined, icon : string\|null) {
437	if (this._debug)	×
438	console.log('Genie sends ' + mediaType + ': '+ url);	×
439	return this.addMessage({ type: mediaType as MessageType.AUDIO\|MessageType.VIDEO\|MessageType.PICTURE, url, alt, icon });	×
440	}
441
442	sendRDL(rdl : RDL, icon : string\|null) {
443	if (this._debug)	24!
444	console.log('Genie sends RDL: '+ rdl.callback);	24✔
445	return this.addMessage({ type: MessageType.RDL, rdl, icon });	24✔
446	}
447
448	sendSoundEffect(name : string, exclusive = false, icon : string\|null) {	×
449	if (this._debug)	1!
450	console.log('Genie sends sound effect: '+ name);	1✔
451	return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });	1✔
452	}
453
454	sendChoice(idx : number, title : string) {
455	if (this._expecting !== ValueCategory.MultipleChoice)	21!
456	console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');	×
457	if (this._debug)	21!
458	console.log('Genie sends multiple choice button: '+ title);	21✔
459	return this.addMessage({ type: MessageType.CHOICE, idx, title });	21✔
460	}
461
462	sendButton(title : string, json : string) {
463	if (this._debug)	×
464	console.log('Genie sends generic button: '+ title);	×
465	return this.addMessage({ type: MessageType.BUTTON, json, title });	×
466	}
467
468	sendLink(title : string, url : string, state : ConversationState) {
469	if (this._debug)	4!
470	console.log('Genie sends link: '+ url);	4✔
471	return this.addMessage({ type: MessageType.LINK, url, title, state });	4✔
472	}
473
474	sendNewProgram(program : {
475	uniqueId : string;
476	name : string;
477	code : string;
478	results : Array<Record<string, unknown>>;
479	errors : string[];
480	icon : string\|null;
481	}) {
482	if (this._debug)	81!
483	console.log('Genie executed new program: '+ program.uniqueId);	81✔
484	return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });	81✔
485	}
486
487	async dialogueFinished() {
488	if (!this.inRecordingMode)	100!
489	return;	×
490	await this._log.dialogueFinished();	100✔
491	}
492
493	async turnFinished() {
494	if (!this.inRecordingMode)	206✔
495	return;	2✔
496	await this._log.turnFinished();	204✔
497	}
498
499	async voteLast(vote : 'up'\|'down') {
500	if (!this.inRecordingMode)	86!
501	return;	×
502	await this._log.voteLast(vote);	86✔
503	}
504
505	async commentLast(comment : string) {
506	if (!this.inRecordingMode)	86!
507	return;	×
508	await this._log.commentLast(comment);	86✔
509	}
510
511	updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'\|'user_timestamp'>, value : string) {
512	if (!this.inRecordingMode)	777✔
513	return;	10✔
514	this._log.updateLog(field, value);	767✔
515	}
516
517	readLog() {
518	const readable = Stream.Readable.from(this._log.read());	1✔
519	const serializer = new DialogueSerializer({ annotations: true });	1✔
520	return readable.pipe(serializer);	1✔
521	}
522	}

stanford-oval / genie-toolkit / 2369

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous