2076

pending completion

Build # 2076

Build Type

Pull #619

travis-ci-com

Committed by

web-flow

Commit Message

Merge 5ab4eca2b into e42330361

Pull Request Pull Request #619: Implement stream based STT through almond-cloud WS

Run Details

9903 of 19988 branches covered (49.54%)

Branch coverage included in aggregate %.

5 of 111 new or added lines in 2 files covered. (4.5%)

2 existing lines in 2 files now uncovered.

23435 of 39424 relevant lines covered (59.44%)

494203.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

2.29

/lib/speech/speech_handler.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as Tp from 'thingpedia';
import * as events from 'events';

import SpeechRecognizer from './speech_recognizer';
import SpeechSynthesizer from './speech_synthesizer';
import { MessageType } from '../dialogue-agent/protocol';
import type Conversation from '../dialogue-agent/conversation';
import type AudioController from '../engine/audio_controller';

interface SpeechHandlerOptions {
    nlUrl ?: string;
}

export default class SpeechHandler extends events.EventEmitter {
    private _platform : Tp.BasePlatform;
    private _prefs : Tp.Preferences;
    private _conversation : Conversation;
    private _pulse : Tp.Capabilities.SoundApi;
    private _wakeWordDetector : Tp.Capabilities.WakeWordApi|null;
    private _voiceDetector : Tp.Capabilities.VadApi|null;
    private _systemLock : Tp.Capabilities.SystemLockApi|null;
    private _recognizer : SpeechRecognizer;
    private _tts : SpeechSynthesizer;
    private _currentRequest : any;
    private _started : boolean;
    private _enableVoiceInput : boolean;
    private _enableVoiceOutput : boolean;
    private _stream : any|null = null;
    private _audioController : AudioController;
    private _queuedAudio : string[];

    constructor(conversation : Conversation,
                platform : Tp.BasePlatform,
                options : SpeechHandlerOptions = {
                    nlUrl: 'https://almond-nl.stanford.edu'
                }) {
        super();
        this._platform = platform;
        this._prefs = platform.getSharedPreferences();

        this._conversation = conversation;
        this._audioController = conversation.engine.audio!;

        this._pulse = platform.getCapability('sound')!;
        this._wakeWordDetector = platform.getCapability('wakeword-detector');
        this._voiceDetector = platform.getCapability('voice-detector');
        this._systemLock = platform.getCapability('system-lock');

        if (this._wakeWordDetector) {
            this._wakeWordDetector.on('wakeword', (wakeword : string, buffer : Buffer) => {
                if (this._systemLock && this._systemLock.isActive) {
                    console.log('Ignored wakeword ' + wakeword + ' because the system is locked');
                    return;
                }

                console.log('Wakeword ' + wakeword + ' detected');
                this.emit('wakeword', wakeword);
                this._onDetected(buffer);
            });
        }

        this._recognizer = new SpeechRecognizer({
            locale: this._platform.locale,
            nlUrl: options.nlUrl,
            vad: this._voiceDetector
        });
        this._recognizer.on('error', (e) => {
            this.emit('error', e);
        });
        this._tts = new SpeechSynthesizer(platform, options.nlUrl);

        this._currentRequest = null;
        this._queuedAudio = [];

        this._started = true;
        this._enableVoiceInput = this._prefs.get('enable-voice-input') as boolean ?? true;
        this._enableVoiceOutput = this._prefs.get('enable-voice-output') as boolean ?? true;

        this._prefs.on('changed', (key : string) => {
            if (key === 'enable-voice-input')
                this.setVoiceInput(this._prefs.get('enable-voice-input') as boolean ?? true);
            else if (key === 'enable-voice-output')
                this.setVoiceOutput(this._prefs.get('enable-voice-output') as boolean ?? true);
        });
    }

    setVoiceInput(enable : boolean) : void {
        if (enable === this._enableVoiceInput)
            return;
        this._enableVoiceInput = enable;
        if (this._started && enable)
            this._startVoiceInput();
        else
            this._stopVoiceInput();
    }

    setVoiceOutput(enable : boolean) : void {
        if (enable === this._enableVoiceOutput)
            return;
        this._enableVoiceOutput = enable;
        if (!enable)
            this._tts.clearQueue();
    }

    // called from conversation
    setHypothesis() : void {
        // ignore, this is called from the conversation when it broadcasts the hypothesis
        // to all listeners
    }

    private _waitFinishSpeaking() {
        return new Promise<void>((resolve, reject) => {
            if (!this._tts.speaking) {
                resolve();
                return;
            }

            this._tts.once('done', () => resolve());
        });
    }

    async setExpected(expect : string) : Promise<void> {
        // flush any request to play audio
        if (this._queuedAudio.length) {
            const toPlay = this._queuedAudio;
            this._queuedAudio = [];
            // wait until the agent finishes speaking to start playing audio
            await this._waitFinishSpeaking();
            const cap = this._platform.getCapability('audio-player');
            if (!cap)
                return;

            let player : {
                stop() : Promise<void>
            }|undefined;
            await this._audioController.requestSystemAudio(async () => {
                if (player) {
                    await player.stop();
                    player = undefined;
                }
            });
            player = await cap.play(toPlay);
        }
    }

    async addMessage(message : any) : Promise<void> {
        switch (message.type) {
        case MessageType.COMMAND:
            await this._tts.clearQueue();
            break;

        case MessageType.TEXT:
            if (!this._enableVoiceOutput)
                break;
            await this._tts.say(message.text);
            break;

        case MessageType.SOUND_EFFECT: {
            const soundEffects = this._platform.getCapability('sound-effects');
            if (soundEffects) {
                if (message.exclusive) {
                    // in exclusive mode, we queue the sound effect as if it was
                    // a regular audio URL
                    // this means we'll stop other audio and we will synchronize
                    // with audio messages
                    const url = soundEffects.getURL(message.name);

                    if (url)
                        this._queuedAudio.push(url);
                    else
                        console.log(`Ignored unknown sound effect ${message.name}`);
                } else {
                    this._waitFinishSpeaking().then(() => {
                        return soundEffects.play(message.name);
                    }).catch((e) => {
                        console.error(`Failed to play sound effect: ${e.message}`);
                    });
                }
            } else {
                console.log(`Ignored sound effect ${message.name}: not supported on this platform`);
            }
            break;
        }

        case MessageType.AUDIO:
            this._queuedAudio.push(message.url);
            break;

        // ignore all other message types
        }
    }

    /**
     * Programmatically trigger a wakeword.
     *
     * This can be used to emulate a wakeword with a push button.
     */
    wakeword() : void {
        this.emit('wakeword');
        this._onDetected(Buffer.from([]));
    }

    private _onDetected(buffer : Buffer, mustHaveWakeword = true) {
        // if we already have a request active, ignore the wakeword, we're
        // already streaming the sound to the server
        if (this._currentRequest)
            return;

        this._currentRequest = this._recognizer.request(this._stream, buffer);
        this._currentRequest.on('hypothesis', (hypothesis : string) => {
            this._conversation.setHypothesis(hypothesis);
        });
        this._currentRequest.on('done', (status : string, utterance : string) => {
            this._currentRequest = null;
            if (status === 'Success') {
                console.log('Recognized as "' + utterance + '"');

                if (mustHaveWakeword) {
                    const wakeWordMatch = /^(computer)[,.!]?/i.exec(utterance);
                    if (!wakeWordMatch) {
                        console.log('Ignored because wake-word is missing');
                        this.emit('no-match');
                        return;
                    }
                    // remove the prefix from the utterance so we don't confuse
                    // the model
                    utterance = utterance.substring(wakeWordMatch[0].length).trim();
                }
                // if there is nothing left, start listening again in case
                // the user paused in-between the wakeword and the command
                // in that case, we will not check for the wakeword and remove it
                //if (!utterance) {
                //    this._onDetected(Buffer.from([]), false);
                //    return;
                //}
                this.emit('match');
                this._conversation.setHypothesis('');
                this._conversation.handleCommand(utterance);
            } else if (status === 'NoMatch' || status === 'InitialSilenceTimeout') {
                this.emit('no-match');
            } else {
                console.log('Recognition error: ' + status);
            }
        });
        this._currentRequest.on('error', (error : Error) => {
            this._currentRequest = null;
            this._onError(error);
        });
    }

    private _onError(error : Error) {
        console.log('Error in speech recognition: ' + error.message);
        this._tts.say("Sorry, I had an error understanding your speech: " + error.message);
    }

    start() : void {
        this._conversation.addOutput(this, false);
        this._started = true;

        if (this._enableVoiceInput)
            this._startVoiceInput();
    }

    private _startVoiceInput() {
        this._stream = this._pulse.createRecordStream({
            format: 'S16LE',
            rate: 16000,
            channels: 1,
            stream: 'genie-voice-output',
            properties: {
                'media.role': 'voice-assistant',
                'filter.want': 'echo-cancel',
            }
        });

        this._stream!.on('state', (state : string) => {
            console.log('Record stream is now ' + state);
            if (state === 'ready')
                this.emit('ready');
        });

        if (this._voiceDetector)
            this._voiceDetector.setup(16000, 3);

        if (this._wakeWordDetector)
            this._stream!.pipe(this._wakeWordDetector);
    }

    stop() {
        this._conversation.removeOutput(this);
        this._started = false;
        this._stopVoiceInput();
        this._tts.clearQueue();
    }

    private _stopVoiceInput() {
        if (!this._stream)
            return;
        this._stream.unpipe();
        this._stream.end();
        this._stream = null;
        this._recognizer.close();
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as Tp from 'thingpedia';
22	import * as events from 'events';	96✔
23
24	import SpeechRecognizer from './speech_recognizer';	96✔
25	import SpeechSynthesizer from './speech_synthesizer';	96✔
26	import { MessageType } from '../dialogue-agent/protocol';	96✔
27	import type Conversation from '../dialogue-agent/conversation';
28	import type AudioController from '../engine/audio_controller';
29
30	interface SpeechHandlerOptions {
31	nlUrl ?: string;
32	}
33
34	export default class SpeechHandler extends events.EventEmitter {	96✔
35	private _platform : Tp.BasePlatform;
36	private _prefs : Tp.Preferences;
37	private _conversation : Conversation;
38	private _pulse : Tp.Capabilities.SoundApi;
39	private _wakeWordDetector : Tp.Capabilities.WakeWordApi\|null;
40	private _voiceDetector : Tp.Capabilities.VadApi\|null;
41	private _systemLock : Tp.Capabilities.SystemLockApi\|null;
42	private _recognizer : SpeechRecognizer;
43	private _tts : SpeechSynthesizer;
44	private _currentRequest : any;
45	private _started : boolean;
46	private _enableVoiceInput : boolean;
47	private _enableVoiceOutput : boolean;
48	private _stream : any\|null = null;	×
49	private _audioController : AudioController;
50	private _queuedAudio : string[];
51
52	constructor(conversation : Conversation,
53	platform : Tp.BasePlatform,
54	options : SpeechHandlerOptions = {	×
55	nlUrl: 'https://almond-nl.stanford.edu'
56	}) {
57	super();	×
58	this._platform = platform;	×
59	this._prefs = platform.getSharedPreferences();	×
60
61	this._conversation = conversation;	×
62	this._audioController = conversation.engine.audio!;	×
63
64	this._pulse = platform.getCapability('sound')!;	×
65	this._wakeWordDetector = platform.getCapability('wakeword-detector');	×
NEW 66	this._voiceDetector = platform.getCapability('voice-detector');	×
UNCOV 67	this._systemLock = platform.getCapability('system-lock');	×
68
69	if (this._wakeWordDetector) {	×
70	this._wakeWordDetector.on('wakeword', (wakeword : string, buffer : Buffer) => {	×
71	if (this._systemLock && this._systemLock.isActive) {	×
72	console.log('Ignored wakeword ' + wakeword + ' because the system is locked');	×
73	return;	×
74	}
75
76	console.log('Wakeword ' + wakeword + ' detected');	×
77	this.emit('wakeword', wakeword);	×
78	this._onDetected(buffer);	×
79	});
80	}
81
82	this._recognizer = new SpeechRecognizer({	×
83	locale: this._platform.locale,
84	nlUrl: options.nlUrl,
85	vad: this._voiceDetector
86	});
87	this._recognizer.on('error', (e) => {	×
88	this.emit('error', e);	×
89	});
NEW 90	this._tts = new SpeechSynthesizer(platform, options.nlUrl);	×
91
92	this._currentRequest = null;	×
93	this._queuedAudio = [];	×
94
95	this._started = true;	×
96	this._enableVoiceInput = this._prefs.get('enable-voice-input') as boolean ?? true;	×
97	this._enableVoiceOutput = this._prefs.get('enable-voice-output') as boolean ?? true;	×
98
99	this._prefs.on('changed', (key : string) => {	×
100	if (key === 'enable-voice-input')	×
101	this.setVoiceInput(this._prefs.get('enable-voice-input') as boolean ?? true);	×
102	else if (key === 'enable-voice-output')	×
103	this.setVoiceOutput(this._prefs.get('enable-voice-output') as boolean ?? true);	×
104	});
105	}
106
107	setVoiceInput(enable : boolean) : void {
108	if (enable === this._enableVoiceInput)	×
109	return;	×
110	this._enableVoiceInput = enable;	×
111	if (this._started && enable)	×
112	this._startVoiceInput();	×
113	else
114	this._stopVoiceInput();	×
115	}
116
117	setVoiceOutput(enable : boolean) : void {
118	if (enable === this._enableVoiceOutput)	×
119	return;	×
120	this._enableVoiceOutput = enable;	×
121	if (!enable)	×
122	this._tts.clearQueue();	×
123	}
124
125	// called from conversation
126	setHypothesis() : void {
127	// ignore, this is called from the conversation when it broadcasts the hypothesis
128	// to all listeners
129	}
130
131	private _waitFinishSpeaking() {
132	return new Promise<void>((resolve, reject) => {	×
133	if (!this._tts.speaking) {	×
134	resolve();	×
135	return;	×
136	}
137
138	this._tts.once('done', () => resolve());	×
139	});
140	}
141
142	async setExpected(expect : string) : Promise<void> {
143	// flush any request to play audio
144	if (this._queuedAudio.length) {	×
145	const toPlay = this._queuedAudio;	×
146	this._queuedAudio = [];	×
147	// wait until the agent finishes speaking to start playing audio
148	await this._waitFinishSpeaking();	×
149	const cap = this._platform.getCapability('audio-player');	×
150	if (!cap)	×
151	return;	×
152
153	let player : {
154	stop() : Promise<void>
155	}\|undefined;
156	await this._audioController.requestSystemAudio(async () => {	×
157	if (player) {	×
158	await player.stop();	×
159	player = undefined;	×
160	}
161	});
162	player = await cap.play(toPlay);	×
163	}
164	}
165
166	async addMessage(message : any) : Promise<void> {
167	switch (message.type) {	×
168	case MessageType.COMMAND:	×
169	await this._tts.clearQueue();	×
170	break;	×
171
172	case MessageType.TEXT:
173	if (!this._enableVoiceOutput)	×
174	break;	×
175	await this._tts.say(message.text);	×
176	break;	×
177
178	case MessageType.SOUND_EFFECT: {
179	const soundEffects = this._platform.getCapability('sound-effects');	×
180	if (soundEffects) {	×
181	if (message.exclusive) {	×
182	// in exclusive mode, we queue the sound effect as if it was
183	// a regular audio URL
184	// this means we'll stop other audio and we will synchronize
185	// with audio messages
186	const url = soundEffects.getURL(message.name);	×
187
188	if (url)	×
189	this._queuedAudio.push(url);	×
190	else
191	console.log(`Ignored unknown sound effect ${message.name}`);	×
192	} else {
193	this._waitFinishSpeaking().then(() => {	×
194	return soundEffects.play(message.name);	×
195	}).catch((e) => {
196	console.error(`Failed to play sound effect: ${e.message}`);	×
197	});
198	}
199	} else {
200	console.log(`Ignored sound effect ${message.name}: not supported on this platform`);	×
201	}
202	break;	×
203	}
204
205	case MessageType.AUDIO:
206	this._queuedAudio.push(message.url);	×
207	break;	×
208
209	// ignore all other message types
210	}
211	}
212
213	/**
214	* Programmatically trigger a wakeword.
215	*
216	* This can be used to emulate a wakeword with a push button.
217	*/
218	wakeword() : void {
219	this.emit('wakeword');	×
220	this._onDetected(Buffer.from([]));	×
221	}
222
223	private _onDetected(buffer : Buffer, mustHaveWakeword = true) {	×
224	// if we already have a request active, ignore the wakeword, we're
225	// already streaming the sound to the server
226	if (this._currentRequest)	×
227	return;	×
228
229	this._currentRequest = this._recognizer.request(this._stream, buffer);	×
230	this._currentRequest.on('hypothesis', (hypothesis : string) => {	×
231	this._conversation.setHypothesis(hypothesis);	×
232	});
233	this._currentRequest.on('done', (status : string, utterance : string) => {	×
234	this._currentRequest = null;	×
235	if (status === 'Success') {	×
236	console.log('Recognized as "' + utterance + '"');	×
237
238	if (mustHaveWakeword) {	×
239	const wakeWordMatch = /^(computer)[,.!]?/i.exec(utterance);	×
240	if (!wakeWordMatch) {	×
241	console.log('Ignored because wake-word is missing');	×
242	this.emit('no-match');	×
243	return;	×
244	}
245	// remove the prefix from the utterance so we don't confuse
246	// the model
247	utterance = utterance.substring(wakeWordMatch[0].length).trim();	×
248	}
249	// if there is nothing left, start listening again in case
250	// the user paused in-between the wakeword and the command
251	// in that case, we will not check for the wakeword and remove it
252	//if (!utterance) {
253	// this._onDetected(Buffer.from([]), false);
254	// return;
255	//}
256	this.emit('match');	×
257	this._conversation.setHypothesis('');	×
258	this._conversation.handleCommand(utterance);	×
259	} else if (status === 'NoMatch' \|\| status === 'InitialSilenceTimeout') {	×
260	this.emit('no-match');	×
261	} else {
262	console.log('Recognition error: ' + status);	×
263	}
264	});
265	this._currentRequest.on('error', (error : Error) => {	×
266	this._currentRequest = null;	×
267	this._onError(error);	×
268	});
269	}
270
271	private _onError(error : Error) {
272	console.log('Error in speech recognition: ' + error.message);	×
273	this._tts.say("Sorry, I had an error understanding your speech: " + error.message);	×
274	}
275
276	start() : void {
277	this._conversation.addOutput(this, false);	×
278	this._started = true;	×
279
280	if (this._enableVoiceInput)	×
281	this._startVoiceInput();	×
282	}
283
284	private _startVoiceInput() {
285	this._stream = this._pulse.createRecordStream({	×
286	format: 'S16LE',
287	rate: 16000,
288	channels: 1,
289	stream: 'genie-voice-output',
290	properties: {
291	'media.role': 'voice-assistant',
292	'filter.want': 'echo-cancel',
293	}
294	});
295
296	this._stream!.on('state', (state : string) => {	×
297	console.log('Record stream is now ' + state);	×
298	if (state === 'ready')	×
299	this.emit('ready');	×
300	});
301
NEW 302	if (this._voiceDetector)	×
NEW 303	this._voiceDetector.setup(16000, 3);	×
304
305	if (this._wakeWordDetector)	×
306	this._stream!.pipe(this._wakeWordDetector);	×
307	}
308
309	stop() {
310	this._conversation.removeOutput(this);	×
311	this._started = false;	×
312	this._stopVoiceInput();	×
313	this._tts.clearQueue();	×
314	}
315
316	private _stopVoiceInput() {
317	if (!this._stream)	×
318	return;	×
319	this._stream.unpipe();	×
320	this._stream.end();	×
321	this._stream = null;	×
322	this._recognizer.close();	×
323	}
324	}

stanford-oval / genie-toolkit / 2076

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous