2328

pending completion

Build # 2328

Build Type

Pull #769

travis-ci-com

Committed by

web-flow

Commit Message

Merge c20e96ae4 into 18ea81b9c

Pull Request Pull Request #769: Add conversation protocol to synchronize devices

Run Details

10082 of 20324 branches covered (49.61%)

Branch coverage included in aggregate %.

50 of 62 new or added lines in 9 files covered. (80.65%)

1 existing line in 1 file now uncovered.

24030 of 40220 relevant lines covered (59.75%)

343903.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.56

/lib/dialogue-agent/conversation.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as events from 'events';
import * as ThingTalk from 'thingtalk';
import * as Stream from "stream";
import * as Tp from 'thingpedia';

import * as I18n from '../i18n';

import ValueCategory from './value-category';
import { DialogueLoop } from './dialogue-loop';
import { PlatformData, MessageType, Message, RDL } from './protocol';
import { EntityMap } from '../utils/entity-utils';
import * as ThingTalkUtils from '../utils/thingtalk';
import type Engine from '../engine';
import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";
import AppExecutor from '../engine/apps/app_executor';
import ConversationLogger from './logging';
import { ConversationStateRow, LocalTable } from "../engine/db";

const DummyStatistics = {
    hit() {
    }
};

const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes

export interface ConversationOptions {
    nluServerUrl ?: string;
    nlgServerUrl ?: string;
    anonymous ?: boolean;
    debug ?: boolean;
    rng ?: () => number;
    inactivityTimeout ?: number;
    contextResetTimeout ?: number;
    showWelcome ?: boolean;
    deleteWhenInactive ?: boolean;
    log ?: boolean;
    dialogueFlags ?: Record<string, boolean>;
    useConfidence ?: boolean;
    faqModels ?: Record<string, {
        url : string;
        highConfidence ?: number;
        lowConfidence ?: number;
    }>;
    syncDevices ?: boolean;
}

interface Statistics {
    hit(key : string) : void;
}

interface Context {
    code : string[];
    entities : EntityMap;
}

export interface ConversationDelegate {
    setHypothesis(hyp : string) : Promise<void>;
    setExpected(expect : string|null, ctx : Context) : Promise<void>;
    addDevice(uniqueId : string, state : Tp.BaseDevice.DeviceState) : Promise<void>;
    addMessage(msg : Message) : Promise<void>;
}

export interface ConversationState {
    history : Message[];
    dialogueState : Record<string, unknown>;
    lastMessageId : number;
}

/**
 * A single session of conversation in Genie.
 *
 * This object is responsible for maintaining the history of the conversation
 * to support clients reconnecting to the same conversation later, as well
 * as tracking connected clients and inactivity timeouts.
 */
export default class Conversation extends events.EventEmitter {
    // NOTE: The actual conversation logic is in DialogueLoop.
    private _engine : Engine;
    private _conversationId : string;
    private _locale : string;
    _ : (x : string) => string;

    private _stats : Statistics;
    private _options : ConversationOptions;
    private _debug : boolean;
    private _dialogueFlags : Record<string, boolean>;
    rng : () => number;

    private _loop : DialogueLoop;
    private _expecting : ValueCategory|null;
    private _context : Context;

    private _started : boolean;
    private _delegates : Set<ConversationDelegate>;
    private _history : Message[];
    private _nextMsgId : number;
    private _inactivityTimeout : NodeJS.Timeout|null;
    private _inactivityTimeoutSec : number;
    private _contextResetTimeout : NodeJS.Timeout|null;
    private _contextResetTimeoutSec : number;
    private _syncDevices : boolean;
    private _deviceAddedListener : (d : Tp.BaseDevice) => void;

    private _log : ConversationLogger;
    private readonly _conversationStateDB : LocalTable<ConversationStateRow>;

    constructor(engine : Engine,
                conversationId : string,
                options : ConversationOptions = {}) {
        super();
        this._engine = engine;
        this._conversationStateDB = this._engine.db.getLocalTable('conversation_state');

        this._conversationId = conversationId;
        this._locale = this._engine.platform.locale;
        this._ = I18n.get(this._locale).gettext;

        const stats = this._engine.platform.getCapability('statistics');
        if (stats === null)
            this._stats = DummyStatistics;
        else
            this._stats = stats;

        this._options = options;
        this._debug = !!this._options.debug;
        this._dialogueFlags = options.dialogueFlags || {};

        this.rng = options.rng || Math.random;

        this._loop = new DialogueLoop(this, this._engine, {
            nluServerUrl: options.nluServerUrl,
            nlgServerUrl: options.nlgServerUrl,
            faqModels: options.faqModels || {},
            useConfidence: options.useConfidence ?? true,
            debug: this._debug,
            rng: this.rng,
        });
        this._expecting = null;
        this._context = { code: ['null'], entities: {} };
        this._delegates = new Set;
        this._history = [];
        this._nextMsgId = 0;
        this._syncDevices = options.syncDevices ?? false;
        this._deviceAddedListener = (d : Tp.BaseDevice) => {
            this.sendNewDevice(d);
        };
        this._started = false;

        this._inactivityTimeout = null;
        this._inactivityTimeoutSec = options.inactivityTimeout || DEFAULT_CONVERSATION_TTL;
        this._contextResetTimeout = null;
        this._contextResetTimeoutSec = options.contextResetTimeout || this._inactivityTimeoutSec;

        this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);
    }

    get isAnonymous() : boolean {
        return !!this._options.anonymous;
    }

    get id() : string {
        return this._conversationId;
    }

    get engine() : Engine {
        return this._engine;
    }

    get stats() : Statistics {
        return this._stats;
    }

    get history() : Message[] {
        return this._history;
    }

    get inRecordingMode() : boolean {
        return !!this._options.log;
    }

    get dialogueFlags() : Record<string, boolean> {
        return this._dialogueFlags;
    }

    startRecording() {
        this._options.log = true;
    }

    async endRecording() {
        await this._log.dialogueFinished();
        this._options.log = false;
    }

    notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
        return this._loop.dispatchNotify(app, outputType, outputValue);
    }

    notifyError(app : AppExecutor, error : Error) {
        return this._loop.dispatchNotifyError(app, error);
    }

    setExpected(expecting : ValueCategory|null, context : Context) : void {
        this._expecting = expecting;
        this._context = context;
    }

    async start(state ?: ConversationState) : Promise<void> {
        this._resetInactivityTimeout();

        if (this._syncDevices) {
            this._engine.devices.on('device-added', this._deviceAddedListener);
            this._engine.devices.on('device-changed', this._deviceAddedListener);
        }

        if (state) {
            for (const msg of state.history)
                await this.addMessage(msg);
            this._nextMsgId = state.lastMessageId+1;
        }
        this._started = true;

        return this._loop.start(!!this._options.showWelcome,
            state ? state.dialogueState : null);
    }

    async stop() : Promise<void> {
        this._engine.devices.removeListener('device-added', this._deviceAddedListener);
        this._engine.devices.removeListener('device-changed', this._deviceAddedListener);
        return this._loop.stop();
    }

    private _resetInactivityTimeout() {
        // after "options.inactivityTimeout" milliseconds we inform the app that the conversation
        // is inactive (turn off LEDs, close the microphone, etc.)
        if (this._inactivityTimeout)
            clearTimeout(this._inactivityTimeout);
        if (this._inactivityTimeoutSec > 0) {
            this._inactivityTimeout = setTimeout(() => {
                this.emit('inactive');
            }, this._inactivityTimeoutSec);
        }

        // after "options.contextResetTimeout", we reset the context, forgetting the state of the
        // conversation
        if (this._contextResetTimeout)
            clearTimeout(this._contextResetTimeout);
        if (this._contextResetTimeoutSec > 0) {
            this._contextResetTimeout = setTimeout(() => {
                this._loop.reset();
            }, this._contextResetTimeoutSec);
        }
    }

    async addOutput(out : ConversationDelegate, replayHistory = true) {
        this._delegates.add(out);
        if (this._syncDevices) {
            for (const d of this._engine.devices.getAllDevices()) {
                if (!await this._callDelegate(out, (out) => out.addDevice(d.uniqueId!, d.serialize())))
                    return;
            }
        }
        if (replayHistory) {
            for (const msg of this._history) {
                if (!await this._callDelegate(out, (out) => out.addMessage(msg)))
                    return;
            }
        }

        if (this._started) {
            const what = ValueCategory.toString(this._expecting);
            await this._callDelegate(out, (out) => out.setExpected(what, this._context));
        }
    }
    async removeOutput(out : ConversationDelegate) {
        this._delegates.delete(out);
    }

    private async _callDelegate(out : ConversationDelegate, fn : (out : ConversationDelegate) => unknown) {
        try {
            await fn(out);
            return true;
        } catch(e) {
            // delegate disappeared (likely a disconnected websocket)
            this._delegates.delete(out);
            return false;
        }
    }

    private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
        return Promise.all(Array.from(this._delegates).map((out) => this._callDelegate(out, fn)));
    }

    async setHypothesis(hypothesis : string) : Promise<void> {
        await this._callDelegates((out) => out.setHypothesis(hypothesis));
    }

    async sendAskSpecial() : Promise<void> {
        const what = ValueCategory.toString(this._expecting);

        if (this._debug) {
            if (what !== null && what !== 'generic')
                console.log('Genie sends a special request');
            else if (what !== null)
                console.log('Genie expects an answer');
        }

        await this._callDelegates((out) => out.setExpected(what, this._context));
    }

    /**
     * Add a message to the conversation history.
     *
     * This method is exported to inject conversation history from outside.
     */
    async addMessage(msg : Message) {
        if (msg.id !== undefined)
            this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);
        else
            msg.id = this._nextMsgId ++;
        this._history.push(msg);
        if (this._history.length > 30)
            this._history.shift();
        await this._callDelegates((out) => out.addMessage(msg));

        await this.saveState(msg.id);
    }

    async saveState(lastMessageId : number) {
        const conversationState = this.getState();
        const row = {
            history: JSON.stringify(/* FIXME conversationState.history */ []),
            dialogueState: JSON.stringify(conversationState.dialogueState),
            lastMessageId: lastMessageId,
        };
        await this._conversationStateDB.insertOne(this._conversationId, row);
    }

    /**
     * Extract the state from the conversation.
     *
     * This method is provided to save and restore the conversation state,
     * and transfer the conversation state between engines.
     */
    getState() : ConversationState {
        const history = this._history.slice();
        const lastMessageId = history.length > 0 ? history[history.length-1].id : null;
        return {
            history: history,
            dialogueState: this._loop.getState(),
            lastMessageId: lastMessageId ? lastMessageId : 0
        };
    }

    async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        // if the command is just whitespace, ignore it without even adding it to the history
        if (!command.trim())
            return;

        this.stats.hit('sabrina-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received assistant command ' + command);

        await this._loop.handleCommand({ type: 'command', utterance: command, platformData });
    }

    async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;
        this.stats.hit('sabrina-parsed-command');
        this.emit('active');
        this._resetInactivityTimeout();
        if (typeof root === 'string')
            root = JSON.parse(root);
        await this.addMessage({ type: MessageType.COMMAND, command: title || command, json: root });

        if (this._debug)
            console.log('Received pre-parsed assistant command');
        if (root.example_id) {
            this._engine.thingpedia.clickExample(root.example_id).catch((e) => {
                console.error('Failed to record example click: ' + e.message);
            });
        }

        if ('program' in root)
            return this.handleThingTalk(root.program, platformData);

        const { code, entities } = root;
        for (const name in entities) {
            if (name.startsWith('SLOT_')) {
                const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];
                const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);
                const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);
                entities[name] = value;
            }
        }

        const parsed = await ThingTalkUtils.parsePrediction(code, entities, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        }, true);
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {
        this._engine.updateActivity();
        const command = `\\t ${program}`;
        this.stats.hit('sabrina-thingtalk-command');
        this.emit('active');
        this._resetInactivityTimeout();
        await this.addMessage({ type: MessageType.COMMAND, command });
        if (this._debug)
            console.log('Received ThingTalk program');

        const parsed = await ThingTalkUtils.parse(program, {
            thingpediaClient: this._engine.thingpedia,
            schemaRetriever: this._engine.schemas,
            loadMetadata: true
        });
        return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });
    }

    async handlePing() {
        this._engine.updateActivity();
        if (this._debug)
            console.log('Received ping message');
    }

    sendReply(message : string, icon : string|null) {
        if (this._debug)
            console.log('Genie says: ' + message);
        return this.addMessage({ type: MessageType.TEXT, text: message, icon });
    }

    sendMedia(mediaType : 'picture'|'audio'|'video', url : string, alt : string|undefined, icon : string|null) {
        if (this._debug)
            console.log('Genie sends ' + mediaType + ': '+ url);
        return this.addMessage({ type: mediaType as MessageType.AUDIO|MessageType.VIDEO|MessageType.PICTURE, url, alt, icon });
    }

    sendRDL(rdl : RDL, icon : string|null) {
        if (this._debug)
            console.log('Genie sends RDL: '+ rdl.callback);
        return this.addMessage({ type: MessageType.RDL, rdl, icon });
    }

    sendSoundEffect(name : string, exclusive = false, icon : string|null) {
        if (this._debug)
            console.log('Genie sends sound effect: '+ name);
        return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });
    }

    sendChoice(idx : number, title : string) {
        if (this._expecting !== ValueCategory.MultipleChoice)
            console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');
        if (this._debug)
            console.log('Genie sends multiple choice button: '+ title);
        return this.addMessage({ type: MessageType.CHOICE, idx, title });
    }

    sendButton(title : string, json : string) {
        if (this._debug)
            console.log('Genie sends generic button: '+ title);
        return this.addMessage({ type: MessageType.BUTTON, json, title });
    }

    sendLink(title : string, url : string, state : ConversationState) {
        if (this._debug)
            console.log('Genie sends link: '+ url);
        return this.addMessage({ type: MessageType.LINK, url, title, state });
    }

    sendNewProgram(program : {
        uniqueId : string;
        name : string;
        code : string;
        results : Array<Record<string, unknown>>;
        errors : string[];
        icon : string|null;
    }) {
        if (this._debug)
            console.log('Genie executed new program: '+ program.uniqueId);
        return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });
    }

    sendNewDevice(device : Tp.BaseDevice) {
        if (this._debug)
            console.log('Genie sends a new device: '+ device.uniqueId);
        const state = device.serialize();
        return this._callDelegates((out) => out.addDevice(device.uniqueId!, state));
    }

    sendPing() {
        if (this._debug)
            console.log('Genie sends ping');
        return this.addMessage({ type: MessageType.PING });
    }

    async dialogueFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.dialogueFinished();
    }

    async turnFinished() {
        if (!this.inRecordingMode)
            return;
        await this._log.turnFinished();
    }

    async voteLast(vote : 'up'|'down') {
        if (!this.inRecordingMode)
            return;
        await this._log.voteLast(vote);
    }

    async commentLast(comment : string) {
        if (!this.inRecordingMode)
            return;
        await this._log.commentLast(comment);
    }

    updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'|'user_timestamp'>, value : string) {
        if (!this.inRecordingMode)
            return;
        this._log.updateLog(field, value);
    }

    readLog() {
        const readable = Stream.Readable.from(this._log.read());
        const serializer = new DialogueSerializer({ annotations: true });
        return readable.pipe(serializer);
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as events from 'events';	97✔
22	import * as ThingTalk from 'thingtalk';	97✔
23	import * as Stream from "stream";	97✔
24	import * as Tp from 'thingpedia';
25
26	import * as I18n from '../i18n';	97✔
27
28	import ValueCategory from './value-category';	97✔
29	import { DialogueLoop } from './dialogue-loop';	97✔
30	import { PlatformData, MessageType, Message, RDL } from './protocol';	97✔
31	import { EntityMap } from '../utils/entity-utils';
32	import * as ThingTalkUtils from '../utils/thingtalk';	97✔
33	import type Engine from '../engine';
34	import { DialogueSerializer, DialogueTurn } from "../dataset-tools/parsers";	97✔
35	import AppExecutor from '../engine/apps/app_executor';
36	import ConversationLogger from './logging';	97✔
37	import { ConversationStateRow, LocalTable } from "../engine/db";
38
39	const DummyStatistics = {	97✔
40	hit() {
41	}
42	};
43
44	const DEFAULT_CONVERSATION_TTL = 600000; // 10 minutes	97✔
45
46	export interface ConversationOptions {
47	nluServerUrl ?: string;
48	nlgServerUrl ?: string;
49	anonymous ?: boolean;
50	debug ?: boolean;
51	rng ?: () => number;
52	inactivityTimeout ?: number;
53	contextResetTimeout ?: number;
54	showWelcome ?: boolean;
55	deleteWhenInactive ?: boolean;
56	log ?: boolean;
57	dialogueFlags ?: Record<string, boolean>;
58	useConfidence ?: boolean;
59	faqModels ?: Record<string, {
60	url : string;
61	highConfidence ?: number;
62	lowConfidence ?: number;
63	}>;
64	syncDevices ?: boolean;
65	}
66
67	interface Statistics {
68	hit(key : string) : void;
69	}
70
71	interface Context {
72	code : string[];
73	entities : EntityMap;
74	}
75
76	export interface ConversationDelegate {
77	setHypothesis(hyp : string) : Promise<void>;
78	setExpected(expect : string\|null, ctx : Context) : Promise<void>;
79	addDevice(uniqueId : string, state : Tp.BaseDevice.DeviceState) : Promise<void>;
80	addMessage(msg : Message) : Promise<void>;
81	}
82
83	export interface ConversationState {
84	history : Message[];
85	dialogueState : Record<string, unknown>;
86	lastMessageId : number;
87	}
88
89	/**
90	* A single session of conversation in Genie.
91	*
92	* This object is responsible for maintaining the history of the conversation
93	* to support clients reconnecting to the same conversation later, as well
94	* as tracking connected clients and inactivity timeouts.
95	*/
96	export default class Conversation extends events.EventEmitter {	97✔
97	// NOTE: The actual conversation logic is in DialogueLoop.
98	private _engine : Engine;
99	private _conversationId : string;
100	private _locale : string;
101	_ : (x : string) => string;
102
103	private _stats : Statistics;
104	private _options : ConversationOptions;
105	private _debug : boolean;
106	private _dialogueFlags : Record<string, boolean>;
107	rng : () => number;
108
109	private _loop : DialogueLoop;
110	private _expecting : ValueCategory\|null;
111	private _context : Context;
112
113	private _started : boolean;
114	private _delegates : Set<ConversationDelegate>;
115	private _history : Message[];
116	private _nextMsgId : number;
117	private _inactivityTimeout : NodeJS.Timeout\|null;
118	private _inactivityTimeoutSec : number;
119	private _contextResetTimeout : NodeJS.Timeout\|null;
120	private _contextResetTimeoutSec : number;
121	private _syncDevices : boolean;
122	private _deviceAddedListener : (d : Tp.BaseDevice) => void;
123
124	private _log : ConversationLogger;
125	private readonly _conversationStateDB : LocalTable<ConversationStateRow>;
126
127	constructor(engine : Engine,
128	conversationId : string,
129	options : ConversationOptions = {}) {	×
130	super();	4✔
131	this._engine = engine;	4✔
132	this._conversationStateDB = this._engine.db.getLocalTable('conversation_state');	4✔
133
134	this._conversationId = conversationId;	4✔
135	this._locale = this._engine.platform.locale;	4✔
136	this._ = I18n.get(this._locale).gettext;	4✔
137
138	const stats = this._engine.platform.getCapability('statistics');	4✔
139	if (stats === null)	4!
140	this._stats = DummyStatistics;	4✔
141	else
142	this._stats = stats;	×
143
144	this._options = options;	4✔
145	this._debug = !!this._options.debug;	4✔
146	this._dialogueFlags = options.dialogueFlags \|\| {};	4✔
147
148	this.rng = options.rng \|\| Math.random;	4✔
149
150	this._loop = new DialogueLoop(this, this._engine, {	4✔
151	nluServerUrl: options.nluServerUrl,
152	nlgServerUrl: options.nlgServerUrl,
153	faqModels: options.faqModels \|\| {},	7✔
154	useConfidence: options.useConfidence ?? true,	12✔
155	debug: this._debug,
156	rng: this.rng,
157	});
158	this._expecting = null;	4✔
159	this._context = { code: ['null'], entities: {} };	4✔
160	this._delegates = new Set;	4✔
161	this._history = [];	4✔
162	this._nextMsgId = 0;	4✔
163	this._syncDevices = options.syncDevices ?? false;	4✔
164	this._deviceAddedListener = (d : Tp.BaseDevice) => {	4✔
165	this.sendNewDevice(d);	9✔
166	};
167	this._started = false;	4✔
168
169	this._inactivityTimeout = null;	4✔
170	this._inactivityTimeoutSec = options.inactivityTimeout \|\| DEFAULT_CONVERSATION_TTL;	4✔
171	this._contextResetTimeout = null;	4✔
172	this._contextResetTimeoutSec = options.contextResetTimeout \|\| this._inactivityTimeoutSec;	4✔
173
174	this._log = new ConversationLogger(engine.db.getLocalTable('conversation'), this._conversationId);	4✔
175	}
176
177	get isAnonymous() : boolean {
178	return !!this._options.anonymous;	110✔
179	}
180
181	get id() : string {
182	return this._conversationId;	×
183	}
184
185	get engine() : Engine {
186	return this._engine;	×
187	}
188
189	get stats() : Statistics {
190	return this._stats;	219✔
191	}
192
193	get history() : Message[] {
194	return this._history;	×
195	}
196
197	get inRecordingMode() : boolean {
198	return !!this._options.log;	1,225✔
199	}
200
201	get dialogueFlags() : Record<string, boolean> {
202	return this._dialogueFlags;	88✔
203	}
204
205	startRecording() {
206	this._options.log = true;	1✔
207	}
208
209	async endRecording() {
210	await this._log.dialogueFinished();	1✔
211	this._options.log = false;	1✔
212	}
213
214	notify(app : AppExecutor, outputType : string, outputValue : Record<string, unknown>) {
215	return this._loop.dispatchNotify(app, outputType, outputValue);	25✔
216	}
217
218	notifyError(app : AppExecutor, error : Error) {
219	return this._loop.dispatchNotifyError(app, error);	3✔
220	}
221
222	setExpected(expecting : ValueCategory\|null, context : Context) : void {
223	this._expecting = expecting;	223✔
224	this._context = context;	223✔
225	}
226
227	async start(state ?: ConversationState) : Promise<void> {
228	this._resetInactivityTimeout();	3✔
229
230	if (this._syncDevices) {	3✔
231	this._engine.devices.on('device-added', this._deviceAddedListener);	1✔
232	this._engine.devices.on('device-changed', this._deviceAddedListener);	1✔
233	}
234
235	if (state) {	3!
236	for (const msg of state.history)	×
237	await this.addMessage(msg);	×
238	this._nextMsgId = state.lastMessageId+1;	×
239	}
240	this._started = true;	3✔
241
242	return this._loop.start(!!this._options.showWelcome,	3✔
243	state ? state.dialogueState : null);	3!
244	}
245
246	async stop() : Promise<void> {
NEW 247	this._engine.devices.removeListener('device-added', this._deviceAddedListener);	×
NEW 248	this._engine.devices.removeListener('device-changed', this._deviceAddedListener);	×
UNCOV 249	return this._loop.stop();	×
250	}
251
252	private _resetInactivityTimeout() {
253	// after "options.inactivityTimeout" milliseconds we inform the app that the conversation
254	// is inactive (turn off LEDs, close the microphone, etc.)
255	if (this._inactivityTimeout)	215✔
256	clearTimeout(this._inactivityTimeout);	212✔
257	if (this._inactivityTimeoutSec > 0) {	215!
258	this._inactivityTimeout = setTimeout(() => {	215✔
259	this.emit('inactive');	×
260	}, this._inactivityTimeoutSec);
261	}
262
263	// after "options.contextResetTimeout", we reset the context, forgetting the state of the
264	// conversation
265	if (this._contextResetTimeout)	215✔
266	clearTimeout(this._contextResetTimeout);	212✔
267	if (this._contextResetTimeoutSec > 0) {	215!
268	this._contextResetTimeout = setTimeout(() => {	215✔
269	this._loop.reset();	×
270	}, this._contextResetTimeoutSec);
271	}
272	}
273
274	async addOutput(out : ConversationDelegate, replayHistory = true) {	3✔
275	this._delegates.add(out);	3✔
276	if (this._syncDevices) {	3✔
277	for (const d of this._engine.devices.getAllDevices()) {	1✔
278	if (!await this._callDelegate(out, (out) => out.addDevice(d.uniqueId!, d.serialize())))	15!
NEW 279	return;	×
280	}
281	}
282	if (replayHistory) {	3!
283	for (const msg of this._history) {	3✔
NEW 284	if (!await this._callDelegate(out, (out) => out.addMessage(msg)))	×
285	return;	×
286	}
287	}
288
289	if (this._started) {	3✔
290	const what = ValueCategory.toString(this._expecting);	2✔
291	await this._callDelegate(out, (out) => out.setExpected(what, this._context));	2✔
292	}
293	}
294	async removeOutput(out : ConversationDelegate) {
295	this._delegates.delete(out);	×
296	}
297
298	private async _callDelegate(out : ConversationDelegate, fn : (out : ConversationDelegate) => unknown) {
299	try {	709✔
300	await fn(out);	709✔
301	return true;	709✔
302	} catch(e) {
303	// delegate disappeared (likely a disconnected websocket)
NEW 304	this._delegates.delete(out);	×
NEW 305	return false;	×
306	}
307	}
308
309	private _callDelegates(fn : (out : ConversationDelegate) => unknown) {
310	return Promise.all(Array.from(this._delegates).map((out) => this._callDelegate(out, fn)));	706✔
311	}
312
313	async setHypothesis(hypothesis : string) : Promise<void> {
314	await this._callDelegates((out) => out.setHypothesis(hypothesis));	×
315	}
316
317	async sendAskSpecial() : Promise<void> {
318	const what = ValueCategory.toString(this._expecting);	215✔
319
320	if (this._debug) {	215!
321	if (what !== null && what !== 'generic')	215✔
322	console.log('Genie sends a special request');	22✔
323	else if (what !== null)	193✔
324	console.log('Genie expects an answer');	4✔
325	}
326
327	await this._callDelegates((out) => out.setExpected(what, this._context));	215✔
328	}
329
330	/**
331	* Add a message to the conversation history.
332	*
333	* This method is exported to inject conversation history from outside.
334	*/
335	async addMessage(msg : Message) {
336	if (msg.id !== undefined)	482!
337	this._nextMsgId = Math.max(this._nextMsgId, msg.id+1);	×
338	else
339	msg.id = this._nextMsgId ++;	482✔
340	this._history.push(msg);	482✔
341	if (this._history.length > 30)	482✔
342	this._history.shift();	434✔
343	await this._callDelegates((out) => out.addMessage(msg));	482✔
344
345	await this.saveState(msg.id);	482✔
346	}
347
348	async saveState(lastMessageId : number) {
349	const conversationState = this.getState();	482✔
350	const row = {	482✔
351	history: JSON.stringify(/* FIXME conversationState.history */ []),
352	dialogueState: JSON.stringify(conversationState.dialogueState),
353	lastMessageId: lastMessageId,
354	};
355	await this._conversationStateDB.insertOne(this._conversationId, row);	482✔
356	}
357
358	/**
359	* Extract the state from the conversation.
360	*
361	* This method is provided to save and restore the conversation state,
362	* and transfer the conversation state between engines.
363	*/
364	getState() : ConversationState {
365	const history = this._history.slice();	489✔
366	const lastMessageId = history.length > 0 ? history[history.length-1].id : null;	489!
367	return {	489✔
368	history: history,
369	dialogueState: this._loop.getState(),
370	lastMessageId: lastMessageId ? lastMessageId : 0	489✔
371	};
372	}
373
374	async handleCommand(command : string, platformData : PlatformData = {}) : Promise<void> {	10✔
375	this._engine.updateActivity();	10✔
376	// if the command is just whitespace, ignore it without even adding it to the history
377	if (!command.trim())	10!
378	return;	×
379
380	this.stats.hit('sabrina-command');	10✔
381	this.emit('active');	10✔
382	this._resetInactivityTimeout();	10✔
383	await this.addMessage({ type: MessageType.COMMAND, command });	10✔
384	if (this._debug)	10!
385	console.log('Received assistant command ' + command);	10✔
386
387	await this._loop.handleCommand({ type: 'command', utterance: command, platformData });	10✔
388	}
389
390	async handleParsedCommand(root : any, title ?: string, platformData : PlatformData = {}) : Promise<void> {	90✔
391	this._engine.updateActivity();	90✔
392	const command = `\\r ${typeof root === 'string' ? root : JSON.stringify(root)}`;	90!
393	this.stats.hit('sabrina-parsed-command');	90✔
394	this.emit('active');	90✔
395	this._resetInactivityTimeout();	90✔
396	if (typeof root === 'string')	90!
397	root = JSON.parse(root);	×
398	await this.addMessage({ type: MessageType.COMMAND, command: title \|\| command, json: root });	90✔
399
400	if (this._debug)	90!
401	console.log('Received pre-parsed assistant command');	90✔
402	if (root.example_id) {	90!
403	this._engine.thingpedia.clickExample(root.example_id).catch((e) => {	×
404	console.error('Failed to record example click: ' + e.message);	×
405	});
406	}
407
408	if ('program' in root)	90!
409	return this.handleThingTalk(root.program, platformData);	×
410
411	const { code, entities } = root;	90✔
412	for (const name in entities) {	90✔
413	if (name.startsWith('SLOT_')) {	×
414	const slotname = root.slots![parseInt(name.substring('SLOT_'.length))];	×
415	const slotType = ThingTalk.Type.fromString(root.slotTypes![slotname]);	×
416	const value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);	×
417	entities[name] = value;	×
418	}
419	}
420
421	const parsed = await ThingTalkUtils.parsePrediction(code, entities, {	90✔
422	thingpediaClient: this._engine.thingpedia,
423	schemaRetriever: this._engine.schemas,
424	loadMetadata: true
425	}, true);
426	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	90✔
427	}
428
429	async handleThingTalk(program : string, platformData : PlatformData = {}) : Promise<void> {	112✔
430	this._engine.updateActivity();	112✔
431	const command = `\\t ${program}`;	112✔
432	this.stats.hit('sabrina-thingtalk-command');	112✔
433	this.emit('active');	112✔
434	this._resetInactivityTimeout();	112✔
435	await this.addMessage({ type: MessageType.COMMAND, command });	112✔
436	if (this._debug)	112!
437	console.log('Received ThingTalk program');	112✔
438
439	const parsed = await ThingTalkUtils.parse(program, {	112✔
440	thingpediaClient: this._engine.thingpedia,
441	schemaRetriever: this._engine.schemas,
442	loadMetadata: true
443	});
444	return this._loop.handleCommand({ type: 'thingtalk', parsed, platformData });	112✔
445	}
446
447	async handlePing() {
448	this._engine.updateActivity();	×
449	if (this._debug)	×
450	console.log('Received ping message');	×
451	}
452
453	sendReply(message : string, icon : string\|null) {
454	if (this._debug)	130!
455	console.log('Genie says: ' + message);	130✔
456	return this.addMessage({ type: MessageType.TEXT, text: message, icon });	130✔
457	}
458
459	sendMedia(mediaType : 'picture'\|'audio'\|'video', url : string, alt : string\|undefined, icon : string\|null) {
460	if (this._debug)	×
461	console.log('Genie sends ' + mediaType + ': '+ url);	×
462	return this.addMessage({ type: mediaType as MessageType.AUDIO\|MessageType.VIDEO\|MessageType.PICTURE, url, alt, icon });	×
463	}
464
465	sendRDL(rdl : RDL, icon : string\|null) {
466	if (this._debug)	24!
467	console.log('Genie sends RDL: '+ rdl.callback);	24✔
468	return this.addMessage({ type: MessageType.RDL, rdl, icon });	24✔
469	}
470
471	sendSoundEffect(name : string, exclusive = false, icon : string\|null) {	×
472	if (this._debug)	1!
473	console.log('Genie sends sound effect: '+ name);	1✔
474	return this.addMessage({ type: MessageType.SOUND_EFFECT, name, exclusive, icon });	1✔
475	}
476
477	sendChoice(idx : number, title : string) {
478	if (this._expecting !== ValueCategory.MultipleChoice)	21!
479	console.log('UNEXPECTED: sendChoice while not expecting a MultipleChoice');	×
480	if (this._debug)	21!
481	console.log('Genie sends multiple choice button: '+ title);	21✔
482	return this.addMessage({ type: MessageType.CHOICE, idx, title });	21✔
483	}
484
485	sendButton(title : string, json : string) {
486	if (this._debug)	×
487	console.log('Genie sends generic button: '+ title);	×
488	return this.addMessage({ type: MessageType.BUTTON, json, title });	×
489	}
490
491	sendLink(title : string, url : string, state : ConversationState) {
492	if (this._debug)	4!
493	console.log('Genie sends link: '+ url);	4✔
494	return this.addMessage({ type: MessageType.LINK, url, title, state });	4✔
495	}
496
497	sendNewProgram(program : {
498	uniqueId : string;
499	name : string;
500	code : string;
501	results : Array<Record<string, unknown>>;
502	errors : string[];
503	icon : string\|null;
504	}) {
505	if (this._debug)	78!
506	console.log('Genie executed new program: '+ program.uniqueId);	78✔
507	return this.addMessage({ type: MessageType.NEW_PROGRAM, ...program });	78✔
508	}
509
510	sendNewDevice(device : Tp.BaseDevice) {
511	if (this._debug)	9!
512	console.log('Genie sends a new device: '+ device.uniqueId);	9✔
513	const state = device.serialize();	9✔
514	return this._callDelegates((out) => out.addDevice(device.uniqueId!, state));	9✔
515	}
516
517	sendPing() {
518	if (this._debug)	9!
519	console.log('Genie sends ping');	×
520	return this.addMessage({ type: MessageType.PING });	9✔
521	}
522
523	async dialogueFinished() {
524	if (!this.inRecordingMode)	98!
525	return;	×
526	await this._log.dialogueFinished();	98✔
527	}
528
529	async turnFinished() {
530	if (!this.inRecordingMode)	201✔
531	return;	2✔
532	await this._log.turnFinished();	199✔
533	}
534
535	async voteLast(vote : 'up'\|'down') {
536	if (!this.inRecordingMode)	84!
537	return;	×
538	await this._log.voteLast(vote);	84✔
539	}
540
541	async commentLast(comment : string) {
542	if (!this.inRecordingMode)	84!
543	return;	×
544	await this._log.commentLast(comment);	84✔
545	}
546
547	updateLog(field : Exclude<keyof DialogueTurn,'agent_timestamp'\|'user_timestamp'>, value : string) {
548	if (!this.inRecordingMode)	758✔
549	return;	10✔
550	this._log.updateLog(field, value);	748✔
551	}
552
553	readLog() {
554	const readable = Stream.Readable.from(this._log.read());	1✔
555	const serializer = new DialogueSerializer({ annotations: true });	1✔
556	return readable.pipe(serializer);	1✔
557	}
558	}

stanford-oval / genie-toolkit / 2328

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous