stanford-oval / genie-toolkit / 725

pending completion

Build # 725

Build Type

Pull #145

travis-ci-com

Committed by

web-flow

Commit Message

Merge 0f416012d into 4ae9652e1

Pull Request Pull Request #145: Inference mode dialogue templates

Run Details

1290 of 4831 branches covered (26.7%)

Branch coverage included in aggregate %.

75 of 314 new or added lines in 11 files covered. (23.89%)

3 existing lines in 2 files now uncovered.

4030 of 9771 relevant lines covered (41.24%)

15695.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

18.37

/lib/sentence-generator/index.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of ThingTalk
//
// Copyright 2017-2018 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const stream = require('stream');
const events = require('events');

const { parseEntity, makeDummyEntities } = require('../utils');

const $runtime = require('./runtime');
const importGenie = require('../genie-compiler');
const i18n = require('../i18n');
const MultiMap = require('../multimap');
const { ReservoirSampler, } = require('../random');

const BASIC_TARGET_GEN_SIZE = 100000;
const CONTEXTUAL_TARGET_GEN_SIZE = 10000;

class SentenceGenerator extends events.EventEmitter {
    constructor(options) {
        super();
        this._target = require('../languages/' + options.targetLanguage);
        this._templateFiles = options.templateFiles;
        this._options = options;
        this._langPack = i18n.get(options.locale);
        this._postprocess = this._langPack.postprocessSynthetic;
        this._grammar = null;
        this._generator = null;
    }

    get progress() {
        return this._grammar.progress;
    }

    postprocess(sentence, program) {
        return this._postprocess(sentence, program, this._options.rng);
    }

    generate(contexts, callback) {
        this._grammar.generate(contexts, callback);
    }
    generateOne(context) {
        return this._grammar.generateOne(context);
    }

    async initialize() {
        this._grammar = new $runtime.Grammar(this._target, this._options);
        for (let file of this._templateFiles) {
            const compiledTemplate = await importGenie(file);
            this._grammar = await compiledTemplate(this._options, this._langPack, this._grammar);
        }
        this._grammar.on('progress', (value) => {
            this.emit('progress', value);
        });
    }
}

const MINIBATCH_SIZE = 5000;
class ContextualSentenceGenerator extends stream.Transform {
    constructor(options = {}) {
        super({ objectMode: true });
        options.contextual = true;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;

        this._options = options;
        this._idPrefix = options.idPrefix;
        this._debug = options.debug;
        this._target = require('../languages/' + options.targetLanguage);
        this._generator = new SentenceGenerator(options);

        this._minibatch = [];
        this._processed = 0;

        this._initialized = false;
        this._i = 0;
    }

    _output(depth, derivation) {
        const context = derivation.context;
        const program = derivation.value.optimize();
        assert(program !== null); // not-null even after optimize
        let preprocessed = derivation.toString();
        preprocessed = preprocessed.replace(/ +/g, ' ');
        preprocessed = this._generator.postprocess(preprocessed, program);

        const [contextCode, contextEntities] = this._target.serializeNormalized(context.value);
        let sequence;
        try {
            sequence = this._target.serialize(program, contextCode, contextEntities);
        } catch(e) {
            //console.error(context.code.join(' '));
            console.error(preprocessed);
            console.error(program.prettyprint());
            console.error(sequence);

            console.error(program.prettyprint(program).trim());
            this.emit('error', e);
        }

        let id = String(this._i++);
        id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;
        const flags = {
            synthetic: true,
            contextual: true,
        };

        this.push({ depth, id, flags, preprocessed, context: contextCode.join(' '), target_code: sequence.join(' ') });
    }

    async _process(minibatch) {
        if (!this._initialized) {
            await this._generator.initialize();
            this._initialized = true;
        }

        const start = Date.now();
        if (this._debug)
            console.log(`Minibatch ${this._processed}-${this._processed+minibatch.length}`);

        const contexts = await Promise.all(minibatch.map(async (contextCode) => {
            const code = contextCode.split(' ');

            const entities = makeDummyEntities(contextCode);
            const program = await this._target.parse(code, entities, this._options);
            return new $runtime.Context(code, program, entities);
        }));

        this._generator.generate(contexts, this._output.bind(this));

        this._processed += minibatch.length;
        const end = Date.now();
        if (this._debug)
            console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds`);
    }

    _transform(contextExample, encoding, callback) {
        this._minibatch.push(contextExample);
        if (this._minibatch.length < MINIBATCH_SIZE) {
            callback();
            return;
        }

        const minibatch = this._minibatch;
        this._minibatch = [];
        this._process(minibatch).then(callback, callback);
    }

    _flush(callback) {
        if (this._minibatch.length > 0)
            this._process(this._minibatch).then(callback, callback);
        else
            process.nextTick(callback);
    }
}

class BasicSentenceGenerator extends stream.Readable {
    constructor(options = {}) {
        super({ objectMode: true });
        options.contextual = false;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = BASIC_TARGET_GEN_SIZE;
        this._target = require('../languages/' + options.targetLanguage);
        this._generator = new SentenceGenerator(options);
        this._generator.on('progress', (value) => {
            this.emit('progress', value);
        });
        this._iterator = null;

        this._initialization = null;
        this._i = 0;
    }

    _read() {
        if (this._initialization === null)
            this._initialization = this._generator.initialize();

        this._initialization.then(() => {
            this._generator.generate([], this._output.bind(this));
            this.emit('progress', this._generator.progress);
            this.push(null);
        }).catch((e) => {
            console.error(e);
            this.emit('error', e);
        });
    }

    _output(depth, derivation) {
        let program = derivation.value.optimize();
        assert(program !== null); // not-null even after optimize
        let preprocessed = derivation.toString();
        preprocessed = preprocessed.replace(/ +/g, ' ');
        preprocessed = this._generator.postprocess(preprocessed, program);
        let sequence;
        try {
            sequence = this._target.serialize(program, [], {});
            //ThingTalk.NNSyntax.fromNN(sequence, {});

            if (sequence.some((t) => t.endsWith(':undefined')))
                throw new TypeError(`Generated undefined type`);
        } catch(e) {
            console.error(preprocessed);
            console.error(String(program));
            console.error(sequence);

            console.error(program.prettyprint().trim());
            this.emit('error', e);
        }
        let id = String(this._i++);
        id = depth + '000000000'.substring(0,9-id.length) + id;
        const flags = {
            synthetic: true
        };
        return this.push({ depth, id, flags, preprocessed, target_code: sequence.join(' ') });
    }
}

class PartialDialogue {
    constructor(context = null, turns = [], constants = {}, execState = undefined) {
        this.context = context;
        this.turns = turns;
        this.constants = constants;
        this.execState = undefined;
    }
}

function mergeConstants(constants, sentence) {
    for (let token of sentence.split(' ')) {
        const entity = parseEntity(token);
        if (entity === null)
            continue;

        const [type, index] = entity;
        constants[type] = Math.max(constants[type] || 0, index + 1);
    }
}

/**
 * Generate a minibatch of dialogues.
 *
 * This object is created afresh for every minibatch.
 */
class MinibatchDialogueGenerator {
    constructor(sentenceGenerator, targetLanguage, dialogueAgent, options, minibatchIdx) {
        this._sentenceGenerator = sentenceGenerator;
        this._target = targetLanguage;
        this._dialogueAgent = dialogueAgent;
        this._minibatchSize = options.minibatchSize;
        this._rng = options.rng;
        this._options = options;

        this._minibatchIdx = minibatchIdx;
        this._turnIdx = 0;

        this._debug = true;

        this._partialDialogs = new ReservoirSampler(Math.ceil(this._minibatchSize * 0.1), this._rng);
        this._emptyDialogue = new PartialDialogue();
        this._maybeAddPartialDialog(this._emptyDialogue);
        this._completeDialogs = new ReservoirSampler(this._minibatchSize, this._rng);
    }

    _maybeAddCompleteDialog(dlg) {
        assert(dlg.turns.length > 0);
        this._completeDialogs.add(dlg.turns);
    }

    _maybeAddPartialDialog(dlg) {
        const discarded = this._partialDialogs.add(dlg);
        if (discarded !== undefined)
            this._maybeAddCompleteDialog(discarded);
        return discarded !== dlg;
    }

    _addTurn(continuations, derivation) {
        const dlg = derivation.context.dlg;

        let [agentState, userState] = derivation.value;
        let agent, agent_target, user, user_target;
        let newConstants = {};
        Object.assign(newConstants, dlg.constants);
        if (dlg.context === null) {
            // first turn in the dialogue
            assert(dlg === this._emptyDialogue);
            assert(agentState === null);
            userState = userState.optimize();
            assert(userState !== null); // not-null even after optimize
            let turnString = derivation.toString();
            turnString = turnString.replace(/ +/g, ' ');
            turnString = this._sentenceGenerator.postprocess(turnString, userState);
            mergeConstants(newConstants, turnString);

            agent = '';
            agent_target = null;
            user = turnString;
            user_target = userState.prettyprint();
        } else {
            agentState = agentState.optimize();
            userState = userState.optimize();
            assert(agentState !== null); // not-null even after optimize
            assert(userState !== null); // not-null even after optimize
            let turnString = derivation.toString();
            turnString = turnString.replace(/ +/g, ' ');
            turnString = this._sentenceGenerator.postprocess(turnString, userState);
            mergeConstants(newConstants, turnString);

            [agent, user] = turnString.split(' <sep> ');
            assert(agent);
            assert(user);

            const agentPrediction = this._target.computePrediction(dlg.context, agentState, 'agent');
            agent_target = agentPrediction.prettyprint();

            const userPrediction = this._target.computePrediction(agentState, userState, 'user');
            user_target = userPrediction.prettyprint();
        }

        continuations.put(dlg, {
            userState,
            newTurn: {
                context: dlg.context !== null ? dlg.context.prettyprint() : null,
                agent,
                agent_target,
                user,
                user_target
            },
            newConstants: {}
        });
    }

    _tryAddTurn(continuations, derivation) {
        try {
            this._addTurn(continuations, derivation);
        } catch(e) {
            console.error(derivation.toString());
            console.error(derivation.value);
            throw e;
        }
    }

    async nextTurn() {
        this._turnIdx++;
        const start = Date.now();
        if (this._debug)
            console.log(`Minibatch ${this._minibatchIdx}, turn ${this._turnIdx}`);

        const partials = this._partialDialogs.sampled;
        this._partialDialogs.reset();

        const continuations = new MultiMap;
        if (this._minibatchSize === 1) {
            assert(partials.length <= 1);
            if (partials.length > 0) {
                const derivation = this._sentenceGenerator.generateOne(partials[0]);
                if (derivation !== undefined)
                    this._tryAddTurn(continuations, derivation);
            }
        } else {
            this._sentenceGenerator.generate(partials, (depth, derivation) => {
                this._tryAddTurn(continuations, derivation);
            });
        }

        for (let i = 0; i < partials.length; i++) {
            const dlg = partials[i];
            const ourContinuations = continuations.get(dlg);

            if (ourContinuations.length === 0) {
                // if we have no continuations, mark this dialog as complete
                this._maybeAddCompleteDialog(dlg);
            } else {
                for (let { userState, newTurn, newConstants } of ourContinuations) {
                    const newDialogue = new PartialDialogue(dlg.context, dlg.turns.concat([newTurn]),
                        newConstants, dlg.execState);

                    if (this._maybeAddPartialDialog(newDialogue)) {
                        const [newContext, newExecState] = await this._dialogueAgent.execute(userState, dlg.execState);
                        newDialogue.context = newContext;
                        newDialogue.execState = newExecState;
                    }
                }
            }
        }

        const end = Date.now();
        if (this._debug) {
            console.log(`Produced ${this._partialDialogs.counter} partial dialogs this turn`);
            console.log(`Produced ${this._completeDialogs.counter} complete dialogs so far`);
            console.log(`Turn took ${Math.round((end-start)/1000)} seconds`);
        }
    }

    complete() {
        for (let dlg of this._partialDialogs)
            this._maybeAddCompleteDialog(dlg);

        assert(this._completeDialogs.length > 0);
        return this._completeDialogs;
    }
}

class DialogueGenerator extends stream.Readable {
    constructor(options) {
        super({ objectMode: true });

        this._i = 0;
        this._targetSize = options.targetSize;
        this._minibatchSize = options.minibatchSize;

        this._options = options;
        this._idPrefix = options.idPrefix || '';
        this._debug = options.debug;
        this._rng = options.rng;

        options.contextual = true;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;
        if (options.maxConstants === undefined)
            options.maxConstants = 5;
        this._target = require('../languages/' + options.targetLanguage);
        this._generator = new SentenceGenerator(options);

        this._initialized = false;
        this._dialogueAgent = this._target.createSimulator(options);
        this._minibatchIdx = 0;
    }

    async _generateMinibatch() {
        if (!this._initialized) {
            await this._generator.initialize();
            this._initialized = true;
        }

        const start = Date.now();
        let counter = 0;
        try {
            const generator = new MinibatchDialogueGenerator(this._generator, this._target, this._dialogueAgent, this._options, this._minibatchIdx++);

            for (let turn = 0; turn < this._options.maxTurns; turn++)
                await generator.nextTurn();

            const generated = generator.complete();
            for (let turns of generated) {
                const dlg = {
                    id: this._idPrefix + '' + this._i++,
                    turns
                };
                this.push(dlg);
                counter ++;
                if (this._i >= this._targetSize) {
                    this.push(null);
                    return;
                }
            }
        } finally {
            const end = Date.now();
            if (this._debug)
                console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds and produced ${counter} dialogues`);
        }
    }

    _read() {
        if (this._i >= this._targetSize) {
            this.push(null);
            return;
        }

        this._generateMinibatch().catch((e) => {
            this.emit('error', e);
        });
    }
}

module.exports = {
    // low-level interface
    SentenceGenerator,

    // stream interface for batch generation
    BasicSentenceGenerator,
    ContextualSentenceGenerator,
    DialogueGenerator
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of ThingTalk
4	//
5	// Copyright 2017-2018 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	48✔
13	const stream = require('stream');	48✔
14	const events = require('events');	48✔
15
16	const { parseEntity, makeDummyEntities } = require('../utils');	48✔
17
18	const $runtime = require('./runtime');	48✔
19	const importGenie = require('../genie-compiler');	48✔
20	const i18n = require('../i18n');	48✔
21	const MultiMap = require('../multimap');	48✔
22	const { ReservoirSampler, } = require('../random');	48✔
23
24	const BASIC_TARGET_GEN_SIZE = 100000;	48✔
25	const CONTEXTUAL_TARGET_GEN_SIZE = 10000;	48✔
26
27	class SentenceGenerator extends events.EventEmitter {
28	constructor(options) {
29	super();	2✔
30	this._target = require('../languages/' + options.targetLanguage);	2✔
31	this._templateFiles = options.templateFiles;	2✔
32	this._options = options;	2✔
33	this._langPack = i18n.get(options.locale);	2✔
34	this._postprocess = this._langPack.postprocessSynthetic;	2✔
35	this._grammar = null;	2✔
36	this._generator = null;	2✔
37	}
38
39	get progress() {
40	return this._grammar.progress;	2✔
41	}
42
43	postprocess(sentence, program) {
44	return this._postprocess(sentence, program, this._options.rng);	1,200,000✔
45	}
46
47	generate(contexts, callback) {
48	this._grammar.generate(contexts, callback);	2✔
49	}
50	generateOne(context) {
NEW 51	return this._grammar.generateOne(context);	×
52	}
53
54	async initialize() {
55	this._grammar = new $runtime.Grammar(this._target, this._options);	2✔
56	for (let file of this._templateFiles) {	2✔
57	const compiledTemplate = await importGenie(file);	2✔
58	this._grammar = await compiledTemplate(this._options, this._langPack, this._grammar);	2✔
59	}
60	this._grammar.on('progress', (value) => {	2✔
61	this.emit('progress', value);	4,210✔
62	});
63	}
64	}
65
66	const MINIBATCH_SIZE = 5000;	48✔
67	class ContextualSentenceGenerator extends stream.Transform {
68	constructor(options = {}) {	×
69	super({ objectMode: true });	×
70	options.contextual = true;	×
71	if (options.targetPruningSize === undefined)	×
72	options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;	×
73
74	this._options = options;	×
75	this._idPrefix = options.idPrefix;	×
76	this._debug = options.debug;	×
77	this._target = require('../languages/' + options.targetLanguage);	×
NEW 78	this._generator = new SentenceGenerator(options);	×
79
80	this._minibatch = [];	×
81	this._processed = 0;	×
82
83	this._initialized = false;	×
84	this._i = 0;	×
85	}
86
87	_output(depth, derivation) {
88	const context = derivation.context;	×
89	const program = derivation.value.optimize();	×
90	assert(program !== null); // not-null even after optimize	×
91	let preprocessed = derivation.toString();	×
92	preprocessed = preprocessed.replace(/ +/g, ' ');	×
93	preprocessed = this._generator.postprocess(preprocessed, program);	×
94
95	const [contextCode, contextEntities] = this._target.serializeNormalized(context.value);	×
96	let sequence;
97	try {	×
98	sequence = this._target.serialize(program, contextCode, contextEntities);	×
99	} catch(e) {
100	//console.error(context.code.join(' '));
101	console.error(preprocessed);	×
102	console.error(program.prettyprint());	×
103	console.error(sequence);	×
104
105	console.error(program.prettyprint(program).trim());	×
106	this.emit('error', e);	×
107	}
108
109	let id = String(this._i++);	×
110	id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;	×
111	const flags = {	×
112	synthetic: true,
113	contextual: true,
114	};
115
116	this.push({ depth, id, flags, preprocessed, context: contextCode.join(' '), target_code: sequence.join(' ') });	×
117	}
118
119	async _process(minibatch) {
120	if (!this._initialized) {	×
121	await this._generator.initialize();	×
122	this._initialized = true;	×
123	}
124
125	const start = Date.now();	×
126	if (this._debug)	×
127	console.log(`Minibatch ${this._processed}-${this._processed+minibatch.length}`);	×
128
129	const contexts = await Promise.all(minibatch.map(async (contextCode) => {	×
130	const code = contextCode.split(' ');	×
131
132	const entities = makeDummyEntities(contextCode);	×
133	const program = await this._target.parse(code, entities, this._options);	×
134	return new $runtime.Context(code, program, entities);	×
135	}));
136
137	this._generator.generate(contexts, this._output.bind(this));	×
138
139	this._processed += minibatch.length;	×
140	const end = Date.now();	×
141	if (this._debug)	×
142	console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds`);	×
143	}
144
145	_transform(contextExample, encoding, callback) {
146	this._minibatch.push(contextExample);	×
147	if (this._minibatch.length < MINIBATCH_SIZE) {	×
148	callback();	×
149	return;	×
150	}
151
152	const minibatch = this._minibatch;	×
153	this._minibatch = [];	×
154	this._process(minibatch).then(callback, callback);	×
155	}
156
157	_flush(callback) {
158	if (this._minibatch.length > 0)	×
159	this._process(this._minibatch).then(callback, callback);	×
160	else
161	process.nextTick(callback);	×
162	}
163	}
164
165	class BasicSentenceGenerator extends stream.Readable {
166	constructor(options = {}) {	×
167	super({ objectMode: true });	2✔
168	options.contextual = false;	2✔
169	if (options.targetPruningSize === undefined)	2!
170	options.targetPruningSize = BASIC_TARGET_GEN_SIZE;	×
171	this._target = require('../languages/' + options.targetLanguage);	2✔
172	this._generator = new SentenceGenerator(options);	2✔
173	this._generator.on('progress', (value) => {	2✔
174	this.emit('progress', value);	4,210✔
175	});
176	this._iterator = null;	2✔
177
178	this._initialization = null;	2✔
179	this._i = 0;	2✔
180	}
181
182	_read() {
183	if (this._initialization === null)	2!
184	this._initialization = this._generator.initialize();	2✔
185
186	this._initialization.then(() => {	2✔
187	this._generator.generate([], this._output.bind(this));	2✔
188	this.emit('progress', this._generator.progress);	2✔
189	this.push(null);	2✔
190	}).catch((e) => {
191	console.error(e);	×
192	this.emit('error', e);	×
193	});
194	}
195
196	_output(depth, derivation) {
197	let program = derivation.value.optimize();	1,200,000✔
198	assert(program !== null); // not-null even after optimize	1,200,000✔
199	let preprocessed = derivation.toString();	1,200,000✔
200	preprocessed = preprocessed.replace(/ +/g, ' ');	1,200,000✔
201	preprocessed = this._generator.postprocess(preprocessed, program);	1,200,000✔
202	let sequence;
203	try {	1,200,000✔
204	sequence = this._target.serialize(program, [], {});	1,200,000✔
205	//ThingTalk.NNSyntax.fromNN(sequence, {});
206
207	if (sequence.some((t) => t.endsWith(':undefined')))	8,489,740!
208	throw new TypeError(`Generated undefined type`);	×
209	} catch(e) {
210	console.error(preprocessed);	×
211	console.error(String(program));	×
212	console.error(sequence);	×
213
214	console.error(program.prettyprint().trim());	×
215	this.emit('error', e);	×
216	}
217	let id = String(this._i++);	1,200,000✔
218	id = depth + '000000000'.substring(0,9-id.length) + id;	1,200,000✔
219	const flags = {	1,200,000✔
220	synthetic: true
221	};
222	return this.push({ depth, id, flags, preprocessed, target_code: sequence.join(' ') });	1,200,000✔
223	}
224	}
225
226	class PartialDialogue {
227	constructor(context = null, turns = [], constants = {}, execState = undefined) {	×
228	this.context = context;	×
229	this.turns = turns;	×
230	this.constants = constants;	×
231	this.execState = undefined;	×
232	}
233	}
234
235	function mergeConstants(constants, sentence) {
236	for (let token of sentence.split(' ')) {	×
237	const entity = parseEntity(token);	×
238	if (entity === null)	×
239	continue;	×
240
241	const [type, index] = entity;	×
242	constants[type] = Math.max(constants[type] \|\| 0, index + 1);	×
243	}
244	}
245
246	/**
247	* Generate a minibatch of dialogues.
248	*
249	* This object is created afresh for every minibatch.
250	*/
251	class MinibatchDialogueGenerator {
252	constructor(sentenceGenerator, targetLanguage, dialogueAgent, options, minibatchIdx) {
253	this._sentenceGenerator = sentenceGenerator;	×
254	this._target = targetLanguage;	×
255	this._dialogueAgent = dialogueAgent;	×
256	this._minibatchSize = options.minibatchSize;	×
257	this._rng = options.rng;	×
258	this._options = options;	×
259
260	this._minibatchIdx = minibatchIdx;	×
261	this._turnIdx = 0;	×
262
263	this._debug = true;	×
264
NEW 265	this._partialDialogs = new ReservoirSampler(Math.ceil(this._minibatchSize * 0.1), this._rng);	×
266	this._emptyDialogue = new PartialDialogue();	×
267	this._maybeAddPartialDialog(this._emptyDialogue);	×
268	this._completeDialogs = new ReservoirSampler(this._minibatchSize, this._rng);	×
269	}
270
271	_maybeAddCompleteDialog(dlg) {
272	assert(dlg.turns.length > 0);	×
273	this._completeDialogs.add(dlg.turns);	×
274	}
275
276	_maybeAddPartialDialog(dlg) {
277	const discarded = this._partialDialogs.add(dlg);	×
278	if (discarded !== undefined)	×
279	this._maybeAddCompleteDialog(discarded);	×
280	return discarded !== dlg;	×
281	}
282
283	_addTurn(continuations, derivation) {
NEW 284	const dlg = derivation.context.dlg;	×
285
NEW 286	let [agentState, userState] = derivation.value;	×
287	let agent, agent_target, user, user_target;
NEW 288	let newConstants = {};	×
NEW 289	Object.assign(newConstants, dlg.constants);	×
NEW 290	if (dlg.context === null) {	×
291	// first turn in the dialogue
NEW 292	assert(dlg === this._emptyDialogue);	×
NEW 293	assert(agentState === null);	×
NEW 294	userState = userState.optimize();	×
NEW 295	assert(userState !== null); // not-null even after optimize	×
NEW 296	let turnString = derivation.toString();	×
NEW 297	turnString = turnString.replace(/ +/g, ' ');	×
NEW 298	turnString = this._sentenceGenerator.postprocess(turnString, userState);	×
NEW 299	mergeConstants(newConstants, turnString);	×
300
NEW 301	agent = '';	×
NEW 302	agent_target = null;	×
NEW 303	user = turnString;	×
NEW 304	user_target = userState.prettyprint();	×
305	} else {
NEW 306	agentState = agentState.optimize();	×
NEW 307	userState = userState.optimize();	×
NEW 308	assert(agentState !== null); // not-null even after optimize	×
NEW 309	assert(userState !== null); // not-null even after optimize	×
NEW 310	let turnString = derivation.toString();	×
NEW 311	turnString = turnString.replace(/ +/g, ' ');	×
NEW 312	turnString = this._sentenceGenerator.postprocess(turnString, userState);	×
NEW 313	mergeConstants(newConstants, turnString);	×
314
NEW 315	[agent, user] = turnString.split(' <sep> ');	×
NEW 316	assert(agent);	×
NEW 317	assert(user);	×
318
NEW 319	const agentPrediction = this._target.computePrediction(dlg.context, agentState, 'agent');	×
NEW 320	agent_target = agentPrediction.prettyprint();	×
321
NEW 322	const userPrediction = this._target.computePrediction(agentState, userState, 'user');	×
NEW 323	user_target = userPrediction.prettyprint();	×
324	}
325
NEW 326	continuations.put(dlg, {	×
327	userState,
328	newTurn: {
329	context: dlg.context !== null ? dlg.context.prettyprint() : null,	×
330	agent,
331	agent_target,
332	user,
333	user_target
334	},
335	newConstants: {}
336	});
337	}
338
339	_tryAddTurn(continuations, derivation) {
NEW 340	try {	×
NEW 341	this._addTurn(continuations, derivation);	×
342	} catch(e) {
NEW 343	console.error(derivation.toString());	×
NEW 344	console.error(derivation.value);	×
NEW 345	throw e;	×
346	}
347	}
348
349	async nextTurn() {
350	this._turnIdx++;	×
351	const start = Date.now();	×
352	if (this._debug)	×
353	console.log(`Minibatch ${this._minibatchIdx}, turn ${this._turnIdx}`);	×
354
355	const partials = this._partialDialogs.sampled;	×
356	this._partialDialogs.reset();	×
357
358	const continuations = new MultiMap;	×
NEW 359	if (this._minibatchSize === 1) {	×
NEW 360	assert(partials.length <= 1);	×
NEW 361	if (partials.length > 0) {	×
NEW 362	const derivation = this._sentenceGenerator.generateOne(partials[0]);	×
NEW 363	if (derivation !== undefined)	×
NEW 364	this._tryAddTurn(continuations, derivation);	×
365	}
366	} else {
NEW 367	this._sentenceGenerator.generate(partials, (depth, derivation) => {	×
NEW 368	this._tryAddTurn(continuations, derivation);	×
369	});
370	}
371
372	for (let i = 0; i < partials.length; i++) {	×
373	const dlg = partials[i];	×
374	const ourContinuations = continuations.get(dlg);	×
375
376	if (ourContinuations.length === 0) {	×
377	// if we have no continuations, mark this dialog as complete
378	this._maybeAddCompleteDialog(dlg);	×
379	} else {
380	for (let { userState, newTurn, newConstants } of ourContinuations) {	×
381	const newDialogue = new PartialDialogue(dlg.context, dlg.turns.concat([newTurn]),	×
382	newConstants, dlg.execState);
383
384	if (this._maybeAddPartialDialog(newDialogue)) {	×
385	const [newContext, newExecState] = await this._dialogueAgent.execute(userState, dlg.execState);	×
386	newDialogue.context = newContext;	×
387	newDialogue.execState = newExecState;	×
388	}
389	}
390	}
391	}
392
393	const end = Date.now();	×
394	if (this._debug) {	×
395	console.log(`Produced ${this._partialDialogs.counter} partial dialogs this turn`);	×
396	console.log(`Produced ${this._completeDialogs.counter} complete dialogs so far`);	×
397	console.log(`Turn took ${Math.round((end-start)/1000)} seconds`);	×
398	}
399	}
400
401	complete() {
402	for (let dlg of this._partialDialogs)	×
403	this._maybeAddCompleteDialog(dlg);	×
404
405	assert(this._completeDialogs.length > 0);	×
406	return this._completeDialogs;	×
407	}
408	}
409
410	class DialogueGenerator extends stream.Readable {
411	constructor(options) {
412	super({ objectMode: true });	×
413
414	this._i = 0;	×
415	this._targetSize = options.targetSize;	×
416	this._minibatchSize = options.minibatchSize;	×
417
418	this._options = options;	×
419	this._idPrefix = options.idPrefix \|\| '';	×
420	this._debug = options.debug;	×
421	this._rng = options.rng;	×
422
423	options.contextual = true;	×
424	if (options.targetPruningSize === undefined)	×
425	options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;	×
426	if (options.maxConstants === undefined)	×
427	options.maxConstants = 5;	×
NEW 428	this._target = require('../languages/' + options.targetLanguage);	×
UNCOV 429	this._generator = new SentenceGenerator(options);	×
430
431	this._initialized = false;	×
432	this._dialogueAgent = this._target.createSimulator(options);	×
433	this._minibatchIdx = 0;	×
434	}
435
436	async _generateMinibatch() {
437	if (!this._initialized) {	×
438	await this._generator.initialize();	×
439	this._initialized = true;	×
440	}
441
442	const start = Date.now();	×
443	let counter = 0;	×
444	try {	×
445	const generator = new MinibatchDialogueGenerator(this._generator, this._target, this._dialogueAgent, this._options, this._minibatchIdx++);	×
446
447	for (let turn = 0; turn < this._options.maxTurns; turn++)	×
448	await generator.nextTurn();	×
449
450	const generated = generator.complete();	×
451	for (let turns of generated) {	×
452	const dlg = {	×
453	id: this._idPrefix + '' + this._i++,
454	turns
455	};
456	this.push(dlg);	×
457	counter ++;	×
458	if (this._i >= this._targetSize) {	×
459	this.push(null);	×
460	return;	×
461	}
462	}
463	} finally {
464	const end = Date.now();	×
465	if (this._debug)	×
466	console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds and produced ${counter} dialogues`);	×
467	}
468	}
469
470	_read() {
471	if (this._i >= this._targetSize) {	×
472	this.push(null);	×
473	return;	×
474	}
475
476	this._generateMinibatch().catch((e) => {	×
477	this.emit('error', e);	×
478	});
479	}
480	}
481
482	module.exports = {	48✔
483	// low-level interface
484	SentenceGenerator,
485
486	// stream interface for batch generation
487	BasicSentenceGenerator,
488	ContextualSentenceGenerator,
489	DialogueGenerator
490	};

stanford-oval / genie-toolkit / 725

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous