1897

pending completion

Build # 1897

Build Type

Pull #565

travis-ci-com

Committed by

web-flow

Commit Message

Merge d1726809f into 9f3e92e3a

Pull Request Pull Request #565: Close audio output stream when assistant finishes speaking

Run Details

12031 of 25587 branches covered (47.02%)

Branch coverage included in aggregate %.

0 of 8 new or added lines in 2 files covered. (0.0%)

1 existing line in 1 file now uncovered.

28204 of 48242 relevant lines covered (58.46%)

395258.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.33

/lib/speech/speech_synthesizer.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as Tp from 'thingpedia';
import assert from 'assert';
import * as events from 'events';
import * as stream from 'stream';

const URL = 'https://almond-nl.stanford.edu';

enum QueueItemType { SPEECH, ERROR }
type QueueItem = {
    type : QueueItemType.SPEECH;
    buffer : Buffer;
    sampleRate : number;
    numChannels : number;
    text : string;
} | {
    type : QueueItemType.ERROR;
    error : Error;
}

interface SoundOutputStream extends stream.Writable {
    discard() : void;
}

export default class SpeechSynthesizer extends events.EventEmitter {
    private _baseUrl : string;
    private _locale : string;
    private _soundCtx : Tp.Capabilities.SoundApi;
    private _queue : Array<QueueItem|Promise<QueueItem>>;
    private _speaking : boolean;
    private _outputStream : SoundOutputStream|null;
    private _sampleRate : number;
    private _numChannels : number;

    constructor(platform : Tp.BasePlatform, url = URL) {
        super();
        this._baseUrl = url;
        this._locale = platform.locale;
        this._soundCtx = platform.getCapability('sound')!;

        this._queue = [];
        this._speaking = false;

        this._outputStream = null;
        this._sampleRate = 0;
        this._numChannels = 0;
    }

    get speaking() {
        return this._speaking;
    }

    async clearQueue() {
        if (this._outputStream)
            this._outputStream.discard();
        this._queue.length = 0;
    }

    private async _synth(text : string) : Promise<QueueItem> {
        try {
            const [buffer,] = await Tp.Helpers.Http.post(this._baseUrl + '/' + this._locale + '/voice/tts', JSON.stringify({
                text
            }), {
                dataContentType: 'application/json',
                raw: true
            });

            const numChannels = buffer.readInt16LE(22);
            const sampleRate = buffer.readInt32LE(24);
            // check bytes per sample (we only support S16LE format, which is what everybody uses anyway)
            assert.strictEqual(buffer.readInt16LE(32), 2);

            console.log(this._numChannels, this._sampleRate);

            // remove the wav header (44 bytes)
            const sliced = buffer.slice(44, buffer.length);
            console.log(buffer.length, sliced.length);

            return { type: QueueItemType.SPEECH, buffer: sliced, sampleRate, numChannels, text };
        } catch(e) {
            return { type: QueueItemType.ERROR, error: e };
        }
    }

    say(text : string) {
        this._queue.push(this._synth(text));
        if (!this._speaking)
            this._sayNext();
    }

    private _silence() {
        // force flush the buffer with 0.15 second of silence
        // this also causes a pause between the utterances, which sounds natural
        // and slows down the pace
        const bufferLength = 0.15 * this._sampleRate * this._numChannels * 2;
        this._outputStream!.write(Buffer.alloc(bufferLength));
        return 150;
    }

    private _closeOutputStream() {
        if (this._outputStream)
            this._outputStream.end();
        this._outputStream = null;
    }

    private _ensureOutputStream(result : { sampleRate : number, numChannels : number }) {
        if (this._outputStream && this._sampleRate === result.sampleRate
            && this._numChannels === result.numChannels)
            return;
        if (this._outputStream)
            this._outputStream.end();
        this._sampleRate = result.sampleRate;
        this._numChannels = result.numChannels;
        this._outputStream = this._soundCtx.createPlaybackStream({
            format: 'S16LE', // signed 16 bit little endian
            rate: this._sampleRate,
            channels: this._numChannels,
            stream: 'genie-voice-output',
            latency: 100000, // us (= 0.1 s)
            properties: {
                'media.role': 'voice-assistant',
                'filter.want': 'echo-cancel',
            }
        }) as SoundOutputStream;
        this._outputStream.on('drain', () => {
            if (this._queue.length === 0) {
                console.log('Done speaking');
                this.emit('done');
                this._speaking = false;
                this._closeOutputStream();
            }
        });
    }

    private async _sayNext() {
        if (this._queue.length === 0) {
            if (!this._speaking)
                this._closeOutputStream();
            return;
        }
        if (!this._speaking) {
            console.log('Starting to speak...');
            this.emit('speaking');
        }
        this._speaking = true;

        const qitem = await this._queue.shift()!;
        try {
            if (qitem.type === QueueItemType.ERROR) {
                throw qitem.error;
            } else {
                this._ensureOutputStream(qitem);

                const duration = qitem.buffer.length /2 /
                    qitem.sampleRate / qitem.numChannels * 1000;
                console.log('outputstream write for ' + qitem.text + ', delay of ' + duration);
                this._outputStream!.write(qitem.buffer);
                this._silence();
            }
        } catch(e) {
            console.error('Failed to speak: ' + e);
        }

        process.nextTick(() => this._sayNext());
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as Tp from 'thingpedia';	96✔
22	import assert from 'assert';	96✔
23	import * as events from 'events';	96✔
24	import * as stream from 'stream';
25
26	const URL = 'https://almond-nl.stanford.edu';	96✔
27
28	enum QueueItemType { SPEECH, ERROR }	96✔
29	type QueueItem = {
30	type : QueueItemType.SPEECH;
31	buffer : Buffer;
32	sampleRate : number;
33	numChannels : number;
34	text : string;
35	} \| {
36	type : QueueItemType.ERROR;
37	error : Error;
38	}
39
40	interface SoundOutputStream extends stream.Writable {
41	discard() : void;
42	}
43
44	export default class SpeechSynthesizer extends events.EventEmitter {	96✔
45	private _baseUrl : string;
46	private _locale : string;
47	private _soundCtx : Tp.Capabilities.SoundApi;
48	private _queue : Array<QueueItem\|Promise<QueueItem>>;
49	private _speaking : boolean;
50	private _outputStream : SoundOutputStream\|null;
51	private _sampleRate : number;
52	private _numChannels : number;
53
54	constructor(platform : Tp.BasePlatform, url = URL) {	×
55	super();	×
56	this._baseUrl = url;	×
57	this._locale = platform.locale;	×
58	this._soundCtx = platform.getCapability('sound')!;	×
59
60	this._queue = [];	×
61	this._speaking = false;	×
62
63	this._outputStream = null;	×
64	this._sampleRate = 0;	×
65	this._numChannels = 0;	×
66	}
67
68	get speaking() {
69	return this._speaking;	×
70	}
71
72	async clearQueue() {
73	if (this._outputStream)	×
74	this._outputStream.discard();	×
75	this._queue.length = 0;	×
76	}
77
78	private async _synth(text : string) : Promise<QueueItem> {
79	try {	×
80	const [buffer,] = await Tp.Helpers.Http.post(this._baseUrl + '/' + this._locale + '/voice/tts', JSON.stringify({	×
81	text
82	}), {
83	dataContentType: 'application/json',
84	raw: true
85	});
86
87	const numChannels = buffer.readInt16LE(22);	×
88	const sampleRate = buffer.readInt32LE(24);	×
89	// check bytes per sample (we only support S16LE format, which is what everybody uses anyway)
90	assert.strictEqual(buffer.readInt16LE(32), 2);	×
91
92	console.log(this._numChannels, this._sampleRate);	×
93
94	// remove the wav header (44 bytes)
95	const sliced = buffer.slice(44, buffer.length);	×
96	console.log(buffer.length, sliced.length);	×
97
98	return { type: QueueItemType.SPEECH, buffer: sliced, sampleRate, numChannels, text };	×
99	} catch(e) {
100	return { type: QueueItemType.ERROR, error: e };	×
101	}
102	}
103
104	say(text : string) {
105	this._queue.push(this._synth(text));	×
106	if (!this._speaking)	×
107	this._sayNext();	×
108	}
109
110	private _silence() {
111	// force flush the buffer with 0.15 second of silence
112	// this also causes a pause between the utterances, which sounds natural
113	// and slows down the pace
114	const bufferLength = 0.15 * this._sampleRate * this._numChannels * 2;	×
115	this._outputStream!.write(Buffer.alloc(bufferLength));	×
116	return 150;	×
117	}
118
119	private _closeOutputStream() {
NEW 120	if (this._outputStream)	×
NEW 121	this._outputStream.end();	×
NEW 122	this._outputStream = null;	×
123	}
124
125	private _ensureOutputStream(result : { sampleRate : number, numChannels : number }) {
UNCOV 126	if (this._outputStream && this._sampleRate === result.sampleRate	×
127	&& this._numChannels === result.numChannels)
128	return;	×
129	if (this._outputStream)	×
130	this._outputStream.end();	×
131	this._sampleRate = result.sampleRate;	×
132	this._numChannels = result.numChannels;	×
133	this._outputStream = this._soundCtx.createPlaybackStream({	×
134	format: 'S16LE', // signed 16 bit little endian
135	rate: this._sampleRate,
136	channels: this._numChannels,
137	stream: 'genie-voice-output',
138	latency: 100000, // us (= 0.1 s)
139	properties: {
140	'media.role': 'voice-assistant',
141	'filter.want': 'echo-cancel',
142	}
143	}) as SoundOutputStream;
144	this._outputStream.on('drain', () => {	×
145	if (this._queue.length === 0) {	×
146	console.log('Done speaking');	×
147	this.emit('done');	×
148	this._speaking = false;	×
NEW 149	this._closeOutputStream();	×
150	}
151	});
152	}
153
154	private async _sayNext() {
NEW 155	if (this._queue.length === 0) {	×
NEW 156	if (!this._speaking)	×
NEW 157	this._closeOutputStream();	×
158	return;	×
159	}
160	if (!this._speaking) {	×
161	console.log('Starting to speak...');	×
162	this.emit('speaking');	×
163	}
164	this._speaking = true;	×
165
166	const qitem = await this._queue.shift()!;	×
167	try {	×
168	if (qitem.type === QueueItemType.ERROR) {	×
169	throw qitem.error;	×
170	} else {
171	this._ensureOutputStream(qitem);	×
172
173	const duration = qitem.buffer.length /2 /	×
174	qitem.sampleRate / qitem.numChannels * 1000;
175	console.log('outputstream write for ' + qitem.text + ', delay of ' + duration);	×
176	this._outputStream!.write(qitem.buffer);	×
177	this._silence();	×
178	}
179	} catch(e) {
180	console.error('Failed to speak: ' + e);	×
181	}
182
183	process.nextTick(() => this._sayNext());	×
184	}
185	}

stanford-oval / genie-toolkit / 1897

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous