2255

pending completion

Build # 2255

Build Type

Pull #751

travis-ci-com

Committed by

web-flow

Commit Message

Merge 36020483c into 84110df3d

Pull Request Pull Request #751: Save and restore conversation state via db

Run Details

10034 of 20366 branches covered (49.27%)

Branch coverage included in aggregate %.

9 of 14 new or added lines in 3 files covered. (64.29%)

1 existing line in 1 file now uncovered.

23918 of 40133 relevant lines covered (59.6%)

344956.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.66

/lib/dialogue-agent/conversation.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as events from 'events';
import * as ThingTalk from 'thingtalk';
import * as Stream from "stream";

import * as I18n from '../i18n';

import ValueCategory from './value-category';
import { DialogueLoop } from './dialogue-loop';
import { PlatformData, MessageType, Message, RDL } from './protocol';
import { EntityMap } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';
import type Engine from '../engine';
import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";
import AppExecutor from '../engine/apps/app_executor';
import ConversationLogger from './logging';
import { ConversationStateRow, LocalTable } from "../engine/db";

const DummyStatistics = {
    hit() {
    }
};

const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes

export interface ConversationOptions {
    nluServerUrl ?: string;
    nlgServerUrl ?: string;
    anonymous ?: boolean;
    debug ?: boolean;
    rng ?: () => number;
    inactivityTimeout ?: number;
    contextResetTimeout ?: number;
    showWelcome ?: boolean;
    deleteWhenInactive ?: boolean;
    log ?: boolean;
    dialogueFlags ?: Record<string, boolean>;
    useConfidence ?: boolean;
    faqModels ?: Record<string, {
        url : string;
        highConfidence ?: number;
        lowConfidence ?: number;
    }>;
}

interface Statistics {
    hit(key : string) : void;
}

interface Context {
    code : string[];
    entities : EntityMap;
}

export interface ConversationDelegate {
    setHypothesis(hyp : string) : void;
    setExpected(expect : string|null, ctx : Context) : void;
    addMessage(msg : Message) : Promise<void>;
}

export interface ConversationState {
    history : Message[];
    dialogueState : Record<string, unknown>;
    lastMessageId : number;
}

/**
 * A single session of conversation in Genie.
 *
 * This object is responsible for maintaining the history of the conversation
 * to support clients reconnecting to the same conversation later, as well
 * as tracking connected clients and inactivity timeouts.
 */
export default class Conversation extends events.EventEmitter {
    // NOTE: The actual conversation logic is in DialogueLoop.
    private _engine : Engine;
    private _conversationId : string;
    private _locale : string;
    _ : (x : string) => string;

    private _stats : Statistics;
    private _options : ConversationOptions;
    private _debug : boolean;
    private _dialogueFlags : Record<string, boolean>;
    rng : () => number;

    private _loop : DialogueLoop;
    private _expecting : ValueCategory|null;
    private _context : Context;

    private _delegates : Set<ConversationDelegate>;
    private _history : Message[];
    private _nextMsgId : number;
    private _inactivityTimeout : NodeJS.Timeout|null;
    private _inactivityTimeoutSec : number;
    private _contextResetTimeout : NodeJS.Timeout|null;
    private _contextResetTimeoutSec : number;

    private _log : ConversationLogger;
    private readonly _conversationStateDB : LocalTable<ConversationStateRow>;

    constructor(engine : Engine,
                conversationId : string,
                options : ConversationOptions = {}) {
        super();
        this._engine = engine;
        this._conversationStateDB = this._engine.db.getLocalTable('conversationState');

        this._conversationId = conversationId;
        this._locale = this._engine.platform.locale;
        this._ = I18n.get(this._locale).gettext;

        const stats = this._engine.platform.getCapability('statistics');
        if (stats === null)
            this._stats = DummyStatistics;
        else
            this._stats = stats;

        this._options = options;
        this._debug = !!this._options.debug;
        this._dialogueFlags = options.dialogueFlags || {};

        this.rng = options.rng || Math.random;

        this._loop = new DialogueLoop(this, this._engine, {
            nluServerUrl: options.nluServerUrl,
            nlgServerUrl: options.nlgServerUrl,
            faqModels: options.faqModels || {},
            useConfidence: options.useConfidence ?? true,
            debug: this._debug,
            rng: this.rng,
        });
        this._expecting = null;
        this._context = { code: ['null'], entities: {} };
        this._delegates = new Set;
        this._history = [];
        this._nextMsgId = 0;

        this._inactivityTimeout = null;
        this._inactivityTimeoutSec = options.inactivityTimeout || DEFAULT_CONVERSATION_TTL;
        this._contextResetTimeout = null;
        this._contextResetTimeoutSec = options.contextResetTimeout || this._inactivityTimeoutSec;

        this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);
    }

    get isAnonymous() : boolean {
        return !!this._options.anonymous;
    }

    get id() : string {
        return this._conversationId;
    }

    get engine() : Engine {
        return this._engine;
    }

    get stats() : Statistics {
        return this._stats;
    }

    get history() : Message[] {
        return this._history;
    }

    get inRecordingMode() : boolean {
        return !!this._options.log;
    }

    get dialogueFlags() : Record<string, boolean> {
        return this._dialogueFlags;
    }

    startRecording() {
        this._options.log = true;
    }

    async endRecording() {
        await this._log.dialogueFinished();
        this._options.log = false;
    }

    notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
        return this._loop.dispatchNotify(app, outputType, outputValue);
    }

    notifyError(app : AppExecutor, error : Error) {
        return this._loop.dispatchNotifyError(app, error);
    }

    setExpected(expecting : ValueCategory|null, context : Context) : void {
        this._expecting = expecting;
        this._context = context;
    }

    async start(state ?: ConversationState) : Promise<void> {
        this._resetInactivityTimeout();
        if (state) {
            for (const msg of state.history)
                await this.addMessage(msg);
            this._nextMsgId = state.lastMessageId+1;
        }
        return this._loop.start(!!this._options.showWelcome,
            state ? state.dialogueState : null);
    }

    async stop() : Promise<void> {
        return this._loop.stop();
    }

    private _resetInactivityTimeout() {
        // after "options.inactivityTimeout" milliseconds we inform the app that the conversation
        // is inactive (turn off LEDs, close the microphone, etc.)
        if (this._inactivityTimeout)
            clearTimeout(this._inactivityTimeout);
        if (this._inactivityTimeoutSec > 0) {
            this._inactivityTimeout = setTimeout(() => {
                this.emit('inactive');
            }, this._inactivityTimeoutSec);
        }

        // after "options.contextResetTimeout", we reset the context, forgetting the state of the
        // conversation
        if (this._contextResetTimeout)
            clearTimeout(this._contextResetTimeout);
        if (this._contextResetTimeoutSec > 0) {
            this._contextResetTimeout = setTimeout(() => {
                this._loop.reset();
            }, this._contextResetTimeoutSec);
        }
    }

    async addOutput(out : ConversationDelegate, replayHistory = true) {
        this._delegates.add(out);
        if (replayHistory) {
            for (const msg of this._history) {
                try {
                    await out.addMessage(msg);
                } catch(e) {
                    // delegate disappeared immediately (race condition)
                    this._delegates.delete(out);
                    return;
                }
            }
        }
    }
    async removeOutput(out : ConversationDelegate) {
        this._delegates.delete(out);
    }

    private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
        return Promise.all(Array.from(this._delegates).map(async (out) => {
            try {
                await fn(out);
            } catch(e) {
                // delegate disappeared (likely a disconnected websocket)
                this._delegates.delete(out);
            }
        }));
    }

    async setHypothesis(hypothesis : string) : Promise<void> {
        await this._callDelegates((out) => out.setHypothesis(hypothesis));
    }

    async sendAskSpecial() : Promise<void> {
        const what = ValueCategory.toString(this._expecting);

        if (this._debug) {
            if (what !== null && what !== 'generic')
                console.log('Genie sends a special request');
            else if (what !== null)
                console.log('Genie expects an answer');
        }

        await this._callDelegates((out) => out.setExpected(what, this._context));
    }

    /**
     * Add a message to the conversation history.
     *
     * This method is exported to inject conversation history from outside.
     */
    async addMessage(msg : Message) {
        if (msg.id !== undefined)
            this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);
        else
            msg.id = this._nextMsgId ++;
        this._history.push(msg);
        if (this._history.length > 30)
            this._history.shift();
        await this._callDelegates((out) => out.addMessage(msg));

        this.saveState(msg.id);
    }

    saveState(lastMessageId : number) {
        const conversationState = this.getState();
        const row = {
            history: JSON.stringify(conversationState.history),
            dialogueState: JSON.stringify(conversationState.dialogueState),
            lastMessageId: lastMessageId,
        };
        this._conversationStateDB.insertOne(this._conversationId, row);
    }

    /**
     * Extract the state from the conversation.
     *
     * This method is provided to save and restore the conversation state,
     * and transfer the conversation state between engines.
     */
    getState() : ConversationState {
        const history = this._history.slice();
        const lastMessageId = history.length > 0 ? history[history.length-1].id : 0;
        return {
            history: history,
            dialogueState: this._loop.getState(),
            lastMessageId: lastMessageId ? lastMessageId : 0
        };
    }

    async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        // if the command is just whitespace, ignore it without even adding it to the history
        if (!command.trim())
            return;

        this.stats.hit('sabrina-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received assistant command ' + command);

        await this._loop.handleCommand({ type: 'command', utterance: command, platformData });
    }

    async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;
        this.stats.hit('sabrina-parsed-command');
        this.emit('active');
        this._resetInactivityTimeout();
        if (typeof root === 'string')
            root = JSON.parse(root);
        await this.addMessage({ type: MessageType.COMMAND, command: title || command, json: root });

        if (this._debug)
            console.log('Received pre-parsed assistant command');
        if (root.example_id) {
            this._engine.thingpedia.clickExample(root.example_id).catch((e) => {
                console.error('Failed to record example click: ' + e.message);
            });
        }

        if ('program' in root)
            return this.handleThingTalk(root.program, platformData);

        const { code, entities } = root;
        for (const name in entities) {
            if (name.startsWith('SLOT_')) {
                const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];
                const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);
                const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);
                entities[name] = value;
            }
        }

        const parsed = await ThingTalkUtils.parsePrediction(code, entities, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        }, true);
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\t ${program}`;
        this.stats.hit('sabrina-thingtalk-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received ThingTalk program');

        const parsed = await ThingTalkUtils.parse(program, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        });
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    async handlePing() {
        this._engine.updateActivity();
        if (this._debug)
            console.log('Received ping message');
    }

    sendReply(message : string, icon : string|null) {
        if (this._debug)
            console.log('Genie says: ' + message);
        return this.addMessage({ type: MessageType.TEXT, text: message, icon });
    }

    sendMedia(mediaType : 'picture'|'audio'|'video', url : string, alt : string|undefined, icon : string|null) {
        if (this._debug)
            console.log('Genie sends ' + mediaType + ': '+ url);
        return this.addMessage({ type: mediaType as MessageType.AUDIO|MessageType.VIDEO|MessageType.PICTURE, url, alt, icon });
    }

    sendRDL(rdl : RDL, icon : string|null) {
        if (this._debug)
            console.log('Genie sends RDL: '+ rdl.callback);
        return this.addMessage({ type: MessageType.RDL, rdl, icon });
    }

    sendSoundEffect(name : string, exclusive = false, icon : string|null) {
        if (this._debug)
            console.log('Genie sends sound effect: '+ name);
        return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });
    }

    sendChoice(idx : number, title : string) {
        if (this._expecting !== ValueCategory.MultipleChoice)
            console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');
        if (this._debug)
            console.log('Genie sends multiple choice button: '+ title);
        return this.addMessage({ type: MessageType.CHOICE, idx, title });
    }

    sendButton(title : string, json : string) {
        if (this._debug)
            console.log('Genie sends generic button: '+ title);
        return this.addMessage({ type: MessageType.BUTTON, json, title });
    }

    sendLink(title : string, url : string, state : ConversationState) {
        if (this._debug)
            console.log('Genie sends link: '+ url);
        return this.addMessage({ type: MessageType.LINK, url, title, state });
    }

    sendNewProgram(program : {
        uniqueId : string;
        name : string;
        code : string;
        results : Array<Record<string, unknown>>;
        errors : string[];
        icon : string|null;
    }) {
        if (this._debug)
            console.log('Genie executed new program: '+ program.uniqueId);
        return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });
    }

    sendPing() {
        if (this._debug)
            console.log('Genie sends ping');
        return this.addMessage({ type: MessageType.PING });
    }

    async dialogueFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.dialogueFinished();
    }

    async turnFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.turnFinished();
    }

    async voteLast(vote : 'up'|'down') {
        if (!this.inRecordingMode)
            return;
        await this._log.voteLast(vote);
    }

    async commentLast(comment : string) {
        if (!this.inRecordingMode)
            return;
        await this._log.commentLast(comment);
    }

    updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'|'user_timestamp'>, value : string) {
        if (!this.inRecordingMode)
            return;
        this._log.updateLog(field, value);
    }

    readLog() {
        const readable = Stream.Readable.from(this._log.read());
        const serializer = new DialogueSerializer({ annotations: true });
        return readable.pipe(serializer);
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as events from 'events';	97✔
22	import * as ThingTalk from 'thingtalk';	97✔
23	import * as Stream from "stream";	97✔
24
25	import * as I18n from '../i18n';	97✔
26
27	import ValueCategory from './value-category';	97✔
28	import { DialogueLoop } from './dialogue-loop';	97✔
29	import { PlatformData, MessageType, Message, RDL } from './protocol';	97✔
30	import { EntityMap } from '../utils/entity-utils';
31	import * as ThingTalkUtils from '../utils/thingtalk';	97✔
32	import type Engine from '../engine';
33	import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";	97✔
34	import AppExecutor from '../engine/apps/app_executor';
35	import ConversationLogger from './logging';	97✔
36	import { ConversationStateRow, LocalTable } from "../engine/db";
37
38	const DummyStatistics = {	97✔
39	hit() {
40	}
41	};
42
43	const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes	97✔
44
45	export interface ConversationOptions {
46	nluServerUrl ?: string;
47	nlgServerUrl ?: string;
48	anonymous ?: boolean;
49	debug ?: boolean;
50	rng ?: () => number;
51	inactivityTimeout ?: number;
52	contextResetTimeout ?: number;
53	showWelcome ?: boolean;
54	deleteWhenInactive ?: boolean;
55	log ?: boolean;
56	dialogueFlags ?: Record<string, boolean>;
57	useConfidence ?: boolean;
58	faqModels ?: Record<string, {
59	url : string;
60	highConfidence ?: number;
61	lowConfidence ?: number;
62	}>;
63	}
64
65	interface Statistics {
66	hit(key : string) : void;
67	}
68
69	interface Context {
70	code : string[];
71	entities : EntityMap;
72	}
73
74	export interface ConversationDelegate {
75	setHypothesis(hyp : string) : void;
76	setExpected(expect : string\|null, ctx : Context) : void;
77	addMessage(msg : Message) : Promise<void>;
78	}
79
80	export interface ConversationState {
81	history : Message[];
82	dialogueState : Record<string, unknown>;
83	lastMessageId : number;
84	}
85
86	/**
87	* A single session of conversation in Genie.
88	*
89	* This object is responsible for maintaining the history of the conversation
90	* to support clients reconnecting to the same conversation later, as well
91	* as tracking connected clients and inactivity timeouts.
92	*/
93	export default class Conversation extends events.EventEmitter {	97✔
94	// NOTE: The actual conversation logic is in DialogueLoop.
95	private _engine : Engine;
96	private _conversationId : string;
97	private _locale : string;
98	_ : (x : string) => string;
99
100	private _stats : Statistics;
101	private _options : ConversationOptions;
102	private _debug : boolean;
103	private _dialogueFlags : Record<string, boolean>;
104	rng : () => number;
105
106	private _loop : DialogueLoop;
107	private _expecting : ValueCategory\|null;
108	private _context : Context;
109
110	private _delegates : Set<ConversationDelegate>;
111	private _history : Message[];
112	private _nextMsgId : number;
113	private _inactivityTimeout : NodeJS.Timeout\|null;
114	private _inactivityTimeoutSec : number;
115	private _contextResetTimeout : NodeJS.Timeout\|null;
116	private _contextResetTimeoutSec : number;
117
118	private _log : ConversationLogger;
119	private readonly _conversationStateDB : LocalTable<ConversationStateRow>;
120
121	constructor(engine : Engine,
122	conversationId : string,
123	options : ConversationOptions = {}) {	×
124	super();	4✔
125	this._engine = engine;	4✔
126	this._conversationStateDB = this._engine.db.getLocalTable('conversationState');	4✔
127
128	this._conversationId = conversationId;	4✔
129	this._locale = this._engine.platform.locale;	4✔
130	this._ = I18n.get(this._locale).gettext;	4✔
131
132	const stats = this._engine.platform.getCapability('statistics');	4✔
133	if (stats === null)	4!
134	this._stats = DummyStatistics;	4✔
135	else
136	this._stats = stats;	×
137
138	this._options = options;	4✔
139	this._debug = !!this._options.debug;	4✔
140	this._dialogueFlags = options.dialogueFlags \|\| {};	4✔
141
142	this.rng = options.rng \|\| Math.random;	4✔
143
144	this._loop = new DialogueLoop(this, this._engine, {	4✔
145	nluServerUrl: options.nluServerUrl,
146	nlgServerUrl: options.nlgServerUrl,
147	faqModels: options.faqModels \|\| {},	7✔
148	useConfidence: options.useConfidence ?? true,	12✔
149	debug: this._debug,
150	rng: this.rng,
151	});
152	this._expecting = null;	4✔
153	this._context = { code: ['null'], entities: {} };	4✔
154	this._delegates = new Set;	4✔
155	this._history = [];	4✔
156	this._nextMsgId = 0;	4✔
157
158	this._inactivityTimeout = null;	4✔
159	this._inactivityTimeoutSec = options.inactivityTimeout \|\| DEFAULT_CONVERSATION_TTL;	4✔
160	this._contextResetTimeout = null;	4✔
161	this._contextResetTimeoutSec = options.contextResetTimeout \|\| this._inactivityTimeoutSec;	4✔
162
163	this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);	4✔
164	}
165
166	get isAnonymous() : boolean {
167	return !!this._options.anonymous;	109✔
168	}
169
170	get id() : string {
171	return this._conversationId;	×
172	}
173
174	get engine() : Engine {
175	return this._engine;	×
176	}
177
178	get stats() : Statistics {
179	return this._stats;	211✔
180	}
181
182	get history() : Message[] {
183	return this._history;	×
184	}
185
186	get inRecordingMode() : boolean {
187	return !!this._options.log;	1,177✔
188	}
189
190	get dialogueFlags() : Record<string, boolean> {
191	return this._dialogueFlags;	84✔
192	}
193
194	startRecording() {
195	this._options.log = true;	1✔
196	}
197
198	async endRecording() {
199	await this._log.dialogueFinished();	1✔
200	this._options.log = false;	1✔
201	}
202
203	notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
204	return this._loop.dispatchNotify(app, outputType, outputValue);	25✔
205	}
206
207	notifyError(app : AppExecutor, error : Error) {
208	return this._loop.dispatchNotifyError(app, error);	3✔
209	}
210
211	setExpected(expecting : ValueCategory\|null, context : Context) : void {
212	this._expecting = expecting;	215✔
213	this._context = context;	215✔
214	}
215
216	async start(state ?: ConversationState) : Promise<void> {
217	this._resetInactivityTimeout();	3✔
218	if (state) {	3!
219	for (const msg of state.history)	×
220	await this.addMessage(msg);	×
NEW 221	this._nextMsgId = state.lastMessageId+1;	×
222	}
223	return this._loop.start(!!this._options.showWelcome,	3✔
224	state ? state.dialogueState : null);	3!
225	}
226
227	async stop() : Promise<void> {
228	return this._loop.stop();	×
229	}
230
231	private _resetInactivityTimeout() {
232	// after "options.inactivityTimeout" milliseconds we inform the app that the conversation
233	// is inactive (turn off LEDs, close the microphone, etc.)
234	if (this._inactivityTimeout)	207✔
235	clearTimeout(this._inactivityTimeout);	204✔
236	if (this._inactivityTimeoutSec > 0) {	207!
237	this._inactivityTimeout = setTimeout(() => {	207✔
238	this.emit('inactive');	×
239	}, this._inactivityTimeoutSec);
240	}
241
242	// after "options.contextResetTimeout", we reset the context, forgetting the state of the
243	// conversation
244	if (this._contextResetTimeout)	207✔
245	clearTimeout(this._contextResetTimeout);	204✔
246	if (this._contextResetTimeoutSec > 0) {	207!
247	this._contextResetTimeout = setTimeout(() => {	207✔
248	this._loop.reset();	×
249	}, this._contextResetTimeoutSec);
250	}
251	}
252
253	async addOutput(out : ConversationDelegate, replayHistory = true) {	3✔
254	this._delegates.add(out);	3✔
255	if (replayHistory) {	3!
256	for (const msg of this._history) {	3✔
257	try {	×
258	await out.addMessage(msg);	×
259	} catch(e) {
260	// delegate disappeared immediately (race condition)
261	this._delegates.delete(out);	×
262	return;	×
263	}
264	}
265	}
266	}
267	async removeOutput(out : ConversationDelegate) {
268	this._delegates.delete(out);	×
269	}
270
271	private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
272	return Promise.all(Array.from(this._delegates).map(async (out) => {	672✔
273	try {	661✔
274	await fn(out);	661✔
275	} catch(e) {
276	// delegate disappeared (likely a disconnected websocket)
277	this._delegates.delete(out);	×
278	}
279	}));
280	}
281
282	async setHypothesis(hypothesis : string) : Promise<void> {
283	await this._callDelegates((out) => out.setHypothesis(hypothesis));	×
284	}
285
286	async sendAskSpecial() : Promise<void> {
287	const what = ValueCategory.toString(this._expecting);	207✔
288
289	if (this._debug) {	207!
290	if (what !== null && what !== 'generic')	207!
291	console.log('Genie sends a special request');	22✔
292	else if (what !== null)	185!
293	console.log('Genie expects an answer');	4✔
294	}
295
296	await this._callDelegates((out) => out.setExpected(what, this._context));	207✔
297	}
298
299	/**
300	* Add a message to the conversation history.
301	*
302	* This method is exported to inject conversation history from outside.
303	*/
304	async addMessage(msg : Message) {
305	if (msg.id !== undefined)	465!
306	this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);	×
307	else
308	msg.id = this._nextMsgId ++;	465✔
309	this._history.push(msg);	465✔
310	if (this._history.length > 30)	465!
311	this._history.shift();	420✔
312	await this._callDelegates((out) => out.addMessage(msg));	465✔
313
314	this.saveState(msg.id);	465✔
315	}
316
317	saveState(lastMessageId : number) {
318	const conversationState = this.getState();	465✔
319	const row = {	465✔
320	history: JSON.stringify(conversationState.history),
321	dialogueState: JSON.stringify(conversationState.dialogueState),
322	lastMessageId: lastMessageId,
323	};
324	this._conversationStateDB.insertOne(this._conversationId, row);	465✔
325	}
326
327	/**
328	* Extract the state from the conversation.
329	*
330	* This method is provided to save and restore the conversation state,
331	* and transfer the conversation state between engines.
332	*/
333	getState() : ConversationState {
334	const history = this._history.slice();	469✔
335	const lastMessageId = history.length > 0 ? history[history.length-1].id : 0;	469!
336	return {	469✔
337	history: history,
338	dialogueState: this._loop.getState(),
339	lastMessageId: lastMessageId ? lastMessageId : 0	469✔
340	};
341	}
342
343	async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {	10!
344	this._engine.updateActivity();	10✔
345	// if the command is just whitespace, ignore it without even adding it to the history
346	if (!command.trim())	10!
347	return;	×
348
349	this.stats.hit('sabrina-command');	10✔
350	this.emit('active');	10✔
351	this._resetInactivityTimeout();	10✔
352	await this.addMessage({ type: MessageType.COMMAND, command });	10✔
353	if (this._debug)	10!
354	console.log('Received assistant command ' + command);	10✔
355
356	await this._loop.handleCommand({ type: 'command', utterance: command, platformData });	10✔
357	}
358
359	async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {	86!
360	this._engine.updateActivity();	86✔
361	const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;	86!
362	this.stats.hit('sabrina-parsed-command');	86✔
363	this.emit('active');	86✔
364	this._resetInactivityTimeout();	86✔
365	if (typeof root === 'string')	86!
366	root = JSON.parse(root);	×
367	await this.addMessage({ type: MessageType.COMMAND, command: title \|\| command, json: root });	86!
368
369	if (this._debug)	86!
370	console.log('Received pre-parsed assistant command');	86✔
371	if (root.example_id) {	86!
372	this._engine.thingpedia.clickExample(root.example_id).catch((e) => {	×
373	console.error('Failed to record example click: ' + e.message);	×
374	});
375	}
376
377	if ('program' in root)	86!
378	return this.handleThingTalk(root.program, platformData);	×
379
380	const { code, entities } = root;	86✔
381	for (const name in entities) {	86✔
382	if (name.startsWith('SLOT_')) {	×
383	const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];	×
384	const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);	×
385	const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);	×
386	entities[name] = value;	×
387	}
388	}
389
390	const parsed = await ThingTalkUtils.parsePrediction(code, entities, {	86✔
391	thingpediaClient: this._engine.thingpedia,
392	schemaRetriever: this._engine.schemas,
393	loadMetadata: true
394	}, true);
395	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	86✔
396	}
397
398	async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {	108✔
399	this._engine.updateActivity();	108✔
400	const command = `\\t ${program}`;	108✔
401	this.stats.hit('sabrina-thingtalk-command');	108✔
402	this.emit('active');	108✔
403	this._resetInactivityTimeout();	108✔
404	await this.addMessage({ type: MessageType.COMMAND, command });	108✔
405	if (this._debug)	108!
406	console.log('Received ThingTalk program');	108✔
407
408	const parsed = await ThingTalkUtils.parse(program, {	108✔
409	thingpediaClient: this._engine.thingpedia,
410	schemaRetriever: this._engine.schemas,
411	loadMetadata: true
412	});
413	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	108✔
414	}
415
416	async handlePing() {
417	this._engine.updateActivity();	×
418	if (this._debug)	×
419	console.log('Received ping message');	×
420	}
421
422	sendReply(message : string, icon : string\|null) {
423	if (this._debug)	126!
424	console.log('Genie says: ' + message);	126✔
425	return this.addMessage({ type: MessageType.TEXT, text: message, icon });	126✔
426	}
427
428	sendMedia(mediaType : 'picture'\|'audio'\|'video', url : string, alt : string\|undefined, icon : string\|null) {
429	if (this._debug)	×
430	console.log('Genie sends ' + mediaType + ': '+ url);	×
431	return this.addMessage({ type: mediaType as MessageType.AUDIO\|MessageType.VIDEO\|MessageType.PICTURE, url, alt, icon });	×
432	}
433
434	sendRDL(rdl : RDL, icon : string\|null) {
435	if (this._debug)	24!
436	console.log('Genie sends RDL: '+ rdl.callback);	24✔
437	return this.addMessage({ type: MessageType.RDL, rdl, icon });	24✔
438	}
439
440	sendSoundEffect(name : string, exclusive = false, icon : string\|null) {	×
441	if (this._debug)	×
442	console.log('Genie sends sound effect: '+ name);	×
443	return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });	×
444	}
445
446	sendChoice(idx : number, title : string) {
447	if (this._expecting !== ValueCategory.MultipleChoice)	21!
448	console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');	×
449	if (this._debug)	21!
450	console.log('Genie sends multiple choice button: '+ title);	21✔
451	return this.addMessage({ type: MessageType.CHOICE, idx, title });	21✔
452	}
453
454	sendButton(title : string, json : string) {
455	if (this._debug)	×
456	console.log('Genie sends generic button: '+ title);	×
457	return this.addMessage({ type: MessageType.BUTTON, json, title });	×
458	}
459
460	sendLink(title : string, url : string, state : ConversationState) {
461	if (this._debug)	4!
462	console.log('Genie sends link: '+ url);	4✔
463	return this.addMessage({ type: MessageType.LINK, url, title, state });	4✔
464	}
465
466	sendNewProgram(program : {
467	uniqueId : string;
468	name : string;
469	code : string;
470	results : Array<Record<string, unknown>>;
471	errors : string[];
472	icon : string\|null;
473	}) {
474	if (this._debug)	77!
475	console.log('Genie executed new program: '+ program.uniqueId);	77✔
476	return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });	77✔
477	}
478
479	sendPing() {
480	if (this._debug)	9!
481	console.log('Genie sends ping');	×
482	return this.addMessage({ type: MessageType.PING });	9✔
483	}
484
485	async dialogueFinished() {
486	if (!this.inRecordingMode)	94!
487	return;	×
488	await this._log.dialogueFinished();	94✔
489	}
490
491	async turnFinished() {
492	if (!this.inRecordingMode)	193!
493	return;	2✔
494	await this._log.turnFinished();	191✔
495	}
496
497	async voteLast(vote : 'up'\|'down') {
498	if (!this.inRecordingMode)	80!
499	return;	×
500	await this._log.voteLast(vote);	80✔
501	}
502
503	async commentLast(comment : string) {
504	if (!this.inRecordingMode)	80!
505	return;	×
506	await this._log.commentLast(comment);	80✔
507	}
508
509	updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'\|'user_timestamp'>, value : string) {
510	if (!this.inRecordingMode)	730!
511	return;	10✔
512	this._log.updateLog(field, value);	720✔
513	}
514
515	readLog() {
516	const readable = Stream.Readable.from(this._log.read());	1✔
517	const serializer = new DialogueSerializer({ annotations: true });	1✔
518	return readable.pipe(serializer);	1✔
519	}
520	}

stanford-oval / genie-toolkit / 2255

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous