1616

pending completion

Build # 1616

Build Type

Pull #451

travis-ci-com

Committed by

web-flow

Commit Message

Merge 93f5b9ecb into d418bae62

Pull Request Pull Request #451: SpeechHandler: react to changes in audio preferences

Run Details

10752 of 23204 branches covered (46.34%)

Branch coverage included in aggregate %.

0 of 11 new or added lines in 1 file covered. (0.0%)

3 existing lines in 1 file now uncovered.

25233 of 43769 relevant lines covered (57.65%)

526694.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.11

/lib/speech/speech_handler.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as Tp from 'thingpedia';
import * as events from 'events';

import SpeechRecognizer from './speech_recognizer';
import SpeechSynthesizer from './speech_synthesizer';
import { MessageType } from '../dialogue-agent/protocol';
import type Conversation from '../dialogue-agent/conversation';

interface SpeechHandlerOptions {
    subscriptionKey ?: string;
}

export default class SpeechHandler extends events.EventEmitter {
    private _platform : Tp.BasePlatform;
    private _prefs : Tp.Preferences;
    private _conversation : Conversation;
    private _pulse : any;
    private _wakeWordDetector : any;
    private _systemLock : any;
    private _recognizer : SpeechRecognizer;
    private _tts : SpeechSynthesizer;
    private _currentRequest : any;
    private _started : boolean;
    private _enableVoiceInput : boolean;
    private _enableVoiceOutput : boolean;
    private _stream : any|null = null;

    constructor(conversation : Conversation,
                platform : Tp.BasePlatform,
                options : SpeechHandlerOptions = {}) {
        super();
        this._platform = platform;
        this._prefs = platform.getSharedPreferences();

        this._conversation = conversation;

        this._pulse = platform.getCapability('sound');
        this._wakeWordDetector = platform.getCapability('wakeword-detector');
        this._systemLock = platform.getCapability('system-lock');

        this._recognizer = new SpeechRecognizer({
            locale: this._platform.locale,
            subscriptionKey: options.subscriptionKey
        });
        this._recognizer.on('error', (e) => {
            this.emit('error', e);
        });
        this._tts = new SpeechSynthesizer(platform);

        this._currentRequest = null;

        this._started = true;
        this._enableVoiceInput = this._prefs.get('enable-voice-input') as boolean ?? true;
        this._enableVoiceOutput = this._prefs.get('enable-voice-output') as boolean ?? true;

        this._prefs.on('changed', (key : string) => {
            if (key === 'enable-voice-input')
                this.setVoiceInput(this._prefs.get('enable-voice-input') as boolean ?? true);
            else if (key === 'enable-voice-output')
                this.setVoiceOutput(this._prefs.get('enable-voice-output') as boolean ?? true);
        });
    }

    setVoiceInput(enable : boolean) : void {
        if (enable === this._enableVoiceInput)
            return;
        this._enableVoiceInput = enable;
        if (this._started && enable)
            this._startVoiceInput();
        else
            this._stopVoiceInput();
    }

    setVoiceOutput(enable : boolean) : void {
        if (enable === this._enableVoiceOutput)
            return;
        this._enableVoiceOutput = enable;
        if (!enable)
            this._tts.clearQueue();
    }

    // called from conversation
    setHypothesis() : void {
        // ignore, this is called from the conversation when it broadcasts the hypothesis
        // to all listeners
    }
    setExpected(expect : string) : void {
    }

    async addMessage(message : any) : Promise<void> {
        switch (message.type) {
        case MessageType.COMMAND:
            await this._tts.clearQueue();
            break;

        case MessageType.TEXT:
        case MessageType.RESULT:
            if (!this._enableVoiceOutput)
                break;
            await this._tts.say(message.text);
            break;

        case MessageType.RDL:
            if (!this._enableVoiceOutput)
                break;
            await this._tts.say(message.rdl.displayTitle);
            break;

        // ignore all other message types
        }
    }

    /**
     * Programmatically trigger a wakeword.
     *
     * This can be used to emulate a wakeword with a push button.
     */
    wakeword() : void {
        this.emit('wakeword');
        this._onDetected();
    }

    private _onDetected() {
        // if we already have a request active, ignore the wakeword, we're
        // already streaming the sound to the server
        if (this._currentRequest)
            return;

        this._currentRequest = this._recognizer.request(this._stream);
        this._currentRequest.on('hypothesis', (hypothesis : string) => {
            this._conversation.setHypothesis(hypothesis);
        });
        this._currentRequest.on('done', (status : string, utterance : string) => {
            this._currentRequest = null;
            if (status === 'Success') {
                console.log('Recognized as "' + utterance + '"');
                this._conversation.setHypothesis('');
                this._conversation.handleCommand(utterance);
            } else if (status === 'NoMatch') {
                this.emit('no-match');
            } else if (status === 'InitialSilenceTimeout') {
                this.emit('silence');
            } else {
                console.log('Recognition error: ' + status);
            }
        });
        this._currentRequest.on('error', (error : Error) => {
            this._currentRequest = null;
            this._onError(error);
        });
    }

    private _onError(error : Error) {
        console.log('Error in speech recognition: ' + error.message);
        this._tts.say("Sorry, I had an error understanding your speech: " + error.message);
    }

    start() : void {
        this._conversation.addOutput(this, false);
        this._started = true;

        if (this._enableVoiceInput)
            this._startVoiceInput();
    }

    private _startVoiceInput() {
        this._stream = this._pulse.createRecordStream({
            format: 'S16LE',
            rate: 16000,
            channels: 1,
            stream: 'genie-voice-output',
            properties: {
                'media.role': 'voice-assistant',
                'filter.want': 'echo-cancel',
            }
        });

        this._stream!.on('state', (state : string) => {
            console.log('Record stream is now ' + state);
            if (state === 'ready')
                this.emit('ready');
        });

        if (this._wakeWordDetector) {
            this._wakeWordDetector.on('wakeword', (wakeword : string) => {
                if (this._systemLock && this._systemLock.isActive) {
                    console.log('Ignored wakeword ' + wakeword + ' because the system is locked');
                    return;
                }

                console.log('Wakeword ' + wakeword + ' detected');
                this.emit('wakeword', wakeword);
                this._onDetected();
            });
            this._stream!.pipe(this._wakeWordDetector);
        }
    }

    stop() {
        this._conversation.removeOutput(this);
        this._started = false;
        this._stopVoiceInput();
        this._tts.clearQueue();
    }

    private _stopVoiceInput() {
        if (!this._stream)
            return;
        this._stream.end();
        this._stream = null;
        this._recognizer.close();
        if (this._wakeWordDetector)
            this._wakeWordDetector.destroy();
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as Tp from 'thingpedia';
22	import * as events from 'events';	1✔
23
24	import SpeechRecognizer from './speech_recognizer';	1✔
25	import SpeechSynthesizer from './speech_synthesizer';	1✔
26	import { MessageType } from '../dialogue-agent/protocol';	1✔
27	import type Conversation from '../dialogue-agent/conversation';
28
29	interface SpeechHandlerOptions {
30	subscriptionKey ?: string;
31	}
32
33	export default class SpeechHandler extends events.EventEmitter {	1✔
34	private _platform : Tp.BasePlatform;
35	private _prefs : Tp.Preferences;
36	private _conversation : Conversation;
37	private _pulse : any;
38	private _wakeWordDetector : any;
39	private _systemLock : any;
40	private _recognizer : SpeechRecognizer;
41	private _tts : SpeechSynthesizer;
42	private _currentRequest : any;
43	private _started : boolean;
44	private _enableVoiceInput : boolean;
45	private _enableVoiceOutput : boolean;
NEW 46	private _stream : any\|null = null;	×
47
48	constructor(conversation : Conversation,
49	platform : Tp.BasePlatform,
50	options : SpeechHandlerOptions = {}) {	×
51	super();	×
52	this._platform = platform;	×
NEW 53	this._prefs = platform.getSharedPreferences();	×
54
55	this._conversation = conversation;	×
56
57	this._pulse = platform.getCapability('sound');	×
58	this._wakeWordDetector = platform.getCapability('wakeword-detector');	×
59	this._systemLock = platform.getCapability('system-lock');	×
60
61	this._recognizer = new SpeechRecognizer({	×
62	locale: this._platform.locale,
63	subscriptionKey: options.subscriptionKey
64	});
65	this._recognizer.on('error', (e) => {	×
66	this.emit('error', e);	×
67	});
68	this._tts = new SpeechSynthesizer(platform);	×
69
70	this._currentRequest = null;	×
71
72	this._started = true;	×
NEW 73	this._enableVoiceInput = this._prefs.get('enable-voice-input') as boolean ?? true;	×
NEW 74	this._enableVoiceOutput = this._prefs.get('enable-voice-output') as boolean ?? true;	×
75
NEW 76	this._prefs.on('changed', (key : string) => {	×
NEW 77	if (key === 'enable-voice-input')	×
NEW 78	this.setVoiceInput(this._prefs.get('enable-voice-input') as boolean ?? true);	×
NEW 79	else if (key === 'enable-voice-output')	×
NEW 80	this.setVoiceOutput(this._prefs.get('enable-voice-output') as boolean ?? true);	×
81	});
82	}
83
84	setVoiceInput(enable : boolean) : void {
85	if (enable === this._enableVoiceInput)	×
86	return;	×
87	this._enableVoiceInput = enable;	×
88	if (this._started && enable)	×
89	this._startVoiceInput();	×
90	else
91	this._stopVoiceInput();	×
92	}
93
94	setVoiceOutput(enable : boolean) : void {
95	if (enable === this._enableVoiceOutput)	×
96	return;	×
97	this._enableVoiceOutput = enable;	×
98	if (!enable)	×
99	this._tts.clearQueue();	×
100	}
101
102	// called from conversation
103	setHypothesis() : void {
104	// ignore, this is called from the conversation when it broadcasts the hypothesis
105	// to all listeners
106	}
107	setExpected(expect : string) : void {
108	}
109
110	async addMessage(message : any) : Promise<void> {
111	switch (message.type) {	×
112	case MessageType.COMMAND:	×
113	await this._tts.clearQueue();	×
114	break;	×
115
116	case MessageType.TEXT:
117	case MessageType.RESULT:
118	if (!this._enableVoiceOutput)	×
119	break;	×
120	await this._tts.say(message.text);	×
121	break;	×
122
123	case MessageType.RDL:
124	if (!this._enableVoiceOutput)	×
125	break;	×
126	await this._tts.say(message.rdl.displayTitle);	×
127	break;	×
128
129	// ignore all other message types
130	}
131	}
132
133	/**
134	* Programmatically trigger a wakeword.
135	*
136	* This can be used to emulate a wakeword with a push button.
137	*/
138	wakeword() : void {
139	this.emit('wakeword');	×
140	this._onDetected();	×
141	}
142
143	private _onDetected() {
144	// if we already have a request active, ignore the wakeword, we're
145	// already streaming the sound to the server
146	if (this._currentRequest)	×
147	return;	×
148
149	this._currentRequest = this._recognizer.request(this._stream);	×
150	this._currentRequest.on('hypothesis', (hypothesis : string) => {	×
151	this._conversation.setHypothesis(hypothesis);	×
152	});
153	this._currentRequest.on('done', (status : string, utterance : string) => {	×
154	this._currentRequest = null;	×
155	if (status === 'Success') {	×
156	console.log('Recognized as "' + utterance + '"');	×
157	this._conversation.setHypothesis('');	×
158	this._conversation.handleCommand(utterance);	×
159	} else if (status === 'NoMatch') {	×
160	this.emit('no-match');	×
161	} else if (status === 'InitialSilenceTimeout') {	×
162	this.emit('silence');	×
163	} else {
164	console.log('Recognition error: ' + status);	×
165	}
166	});
167	this._currentRequest.on('error', (error : Error) => {	×
168	this._currentRequest = null;	×
169	this._onError(error);	×
170	});
171	}
172
173	private _onError(error : Error) {
174	console.log('Error in speech recognition: ' + error.message);	×
175	this._tts.say("Sorry, I had an error understanding your speech: " + error.message);	×
176	}
177
178	start() : void {
179	this._conversation.addOutput(this, false);	×
180	this._started = true;	×
181
182	if (this._enableVoiceInput)	×
183	this._startVoiceInput();	×
184	}
185
186	private _startVoiceInput() {
187	this._stream = this._pulse.createRecordStream({	×
188	format: 'S16LE',
189	rate: 16000,
190	channels: 1,
191	stream: 'genie-voice-output',
192	properties: {
193	'media.role': 'voice-assistant',
194	'filter.want': 'echo-cancel',
195	}
196	});
197
NEW 198	this._stream!.on('state', (state : string) => {	×
199	console.log('Record stream is now ' + state);	×
200	if (state === 'ready')	×
201	this.emit('ready');	×
202	});
203
204	if (this._wakeWordDetector) {	×
205	this._wakeWordDetector.on('wakeword', (wakeword : string) => {	×
206	if (this._systemLock && this._systemLock.isActive) {	×
207	console.log('Ignored wakeword ' + wakeword + ' because the system is locked');	×
208	return;	×
209	}
210
211	console.log('Wakeword ' + wakeword + ' detected');	×
212	this.emit('wakeword', wakeword);	×
213	this._onDetected();	×
214	});
NEW 215	this._stream!.pipe(this._wakeWordDetector);	×
216	}
217	}
218
219	stop() {
220	this._conversation.removeOutput(this);	×
221	this._started = false;	×
222	this._stopVoiceInput();	×
223	this._tts.clearQueue();	×
224	}
225
226	private _stopVoiceInput() {
227	if (!this._stream)	×
228	return;	×
229	this._stream.end();	×
230	this._stream = null;	×
231	this._recognizer.close();	×
232	if (this._wakeWordDetector)	×
233	this._wakeWordDetector.destroy();	×
234	}
235	}

stanford-oval / genie-toolkit / 1616

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous