stanford-oval / genie-toolkit / 835

pending completion

Build # 835

Build Type

Pull #174

travis-ci-com

Committed by

web-flow

Commit Message

Merge 31da96d11 into a6faf33d9

Pull Request Pull Request #174: Core changes from the dialogue work

Run Details

1371 of 5142 branches covered (26.66%)

Branch coverage included in aggregate %.

26 of 53 new or added lines in 7 files covered. (49.06%)

1 existing line in 1 file now uncovered.

4181 of 10208 relevant lines covered (40.96%)

16198.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.4

/lib/sentence-generator/index.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of ThingTalk
//
// Copyright 2017-2018 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const stream = require('stream');
const events = require('events');

const { parseEntity, makeDummyEntities } = require('../utils');

const $runtime = require('./runtime');
const importGenie = require('../genie-compiler');
const i18n = require('../i18n');
const MultiMap = require('../multimap');
const { ReservoirSampler, } = require('../random');

const BASIC_TARGET_GEN_SIZE = 100000;
const CONTEXTUAL_TARGET_GEN_SIZE = 10000;

class SentenceGenerator extends events.EventEmitter {
    constructor(options) {
        super();
        this._target = require('../languages/' + options.targetLanguage);
        this._templateFiles = options.templateFiles;
        this._options = options;
        this._langPack = i18n.get(options.locale);
        this._postprocess = this._langPack.postprocessSynthetic;
        this._grammar = null;
        this._generator = null;
    }

    get progress() {
        return this._grammar.progress;
    }

    postprocess(sentence, program) {
        return this._postprocess(sentence, program, this._options.rng);
    }

    generate(contexts, callback) {
        this._grammar.generate(contexts, callback);
    }
    generateOne(context) {
        return this._grammar.generateOne(context);
    }

    async initialize() {
        this._grammar = new $runtime.Grammar(this._target, this._options);
        for (let file of this._templateFiles) {
            const compiledTemplate = await importGenie(file);
            this._grammar = await compiledTemplate(this._options, this._langPack, this._grammar);
        }
        this._grammar.on('progress', (value) => {
            this.emit('progress', value);
        });
    }
}

const MINIBATCH_SIZE = 5000;
class ContextualSentenceGenerator extends stream.Transform {
    constructor(options = {}) {
        super({ objectMode: true });
        options.contextual = true;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;

        this._options = options;
        this._idPrefix = options.idPrefix;
        this._debug = options.debug;
        this._target = require('../languages/' + options.targetLanguage);
        this._generator = new SentenceGenerator(options);

        this._minibatch = [];
        this._processed = 0;

        this._initialized = false;
        this._i = 0;
    }

    _output(depth, derivation) {
        const context = derivation.context;
        const program = derivation.value.optimize();
        assert(program !== null); // not-null even after optimize
        let preprocessed = derivation.toString();
        preprocessed = preprocessed.replace(/ +/g, ' ');
        preprocessed = this._generator.postprocess(preprocessed, program);

        const [contextCode, contextEntities] = this._target.serializeNormalized(context.value);
        let sequence;
        try {
            sequence = this._target.serialize(program, contextCode, contextEntities);
        } catch(e) {
            //console.error(context.code.join(' '));
            console.error(preprocessed);
            console.error(program.prettyprint());
            console.error(sequence);

            console.error(program.prettyprint(program).trim());
            this.emit('error', e);
        }

        let id = String(this._i++);
        id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;
        const flags = {
            synthetic: true,
            contextual: true,
        };

        this.push({ depth, id, flags, preprocessed, context: contextCode.join(' '), target_code: sequence.join(' ') });
    }

    async _process(minibatch) {
        if (!this._initialized) {
            await this._generator.initialize();
            this._initialized = true;
        }

        const start = Date.now();
        if (this._debug)
            console.log(`Minibatch ${this._processed}-${this._processed+minibatch.length}`);

        const contexts = await Promise.all(minibatch.map(async (contextCode) => {
            const code = contextCode.split(' ');

            const entities = makeDummyEntities(contextCode);
            const program = await this._target.parse(code, entities, this._options);
            return new $runtime.Context(code, program, entities);
        }));

        this._generator.generate(contexts, this._output.bind(this));

        this._processed += minibatch.length;
        const end = Date.now();
        if (this._debug)
            console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds`);
    }

    _transform(contextExample, encoding, callback) {
        this._minibatch.push(contextExample);
        if (this._minibatch.length < MINIBATCH_SIZE) {
            callback();
            return;
        }

        const minibatch = this._minibatch;
        this._minibatch = [];
        this._process(minibatch).then(callback, callback);
    }

    _flush(callback) {
        if (this._minibatch.length > 0)
            this._process(this._minibatch).then(callback, callback);
        else
            process.nextTick(callback);
    }
}

class BasicSentenceGenerator extends stream.Readable {
    constructor(options = {}) {
        super({ objectMode: true });
        options.contextual = false;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = BASIC_TARGET_GEN_SIZE;
        this._target = require('../languages/' + options.targetLanguage);
        this._idPrefix = options.idPrefix || '';
        this._generator = new SentenceGenerator(options);
        this._generator.on('progress', (value) => {
            this.emit('progress', value);
        });
        this._iterator = null;

        this._initialization = null;
        this._i = 0;
    }

    _read() {
        if (this._initialization === null)
            this._initialization = this._generator.initialize();

        this._initialization.then(() => {
            this._generator.generate([], this._output.bind(this));
            this.emit('progress', this._generator.progress);
            this.push(null);
        }).catch((e) => {
            console.error(e);
            this.emit('error', e);
        });
    }

    _output(depth, derivation) {
        let program = derivation.value.optimize();
        assert(program !== null); // not-null even after optimize
        let preprocessed = derivation.toString();
        preprocessed = preprocessed.replace(/ +/g, ' ');
        preprocessed = this._generator.postprocess(preprocessed, program);
        let sequence;
        try {
            sequence = this._target.serialize(program, [], {});
            //ThingTalk.NNSyntax.fromNN(sequence, {});

            if (sequence.some((t) => t.endsWith(':undefined')))
                throw new TypeError(`Generated undefined type`);
        } catch(e) {
            console.error(preprocessed);
            console.error(String(program));
            console.error(sequence);

            console.error(program.prettyprint().trim());
            this.emit('error', e);
        }
        let id = String(this._i++);
        id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;
        const flags = {
            synthetic: true
        };
        return this.push({ depth, id, flags, preprocessed, target_code: sequence.join(' ') });
    }
}

class PartialDialogue {
    constructor(context = null, turns = [], constants = {}, execState = undefined) {
        this.context = context;
        this.turns = turns;
        this.constants = constants;
        this.execState = undefined;
    }
}

function mergeConstants(constants, sentence) {
    for (let token of sentence.split(' ')) {
        const entity = parseEntity(token);
        if (entity === null)
            continue;

        const [type, index] = entity;
        constants[type] = Math.max(constants[type] || 0, index + 1);
    }
}

const FACTORS = [50, 75, 75, 100];

/**
 * Generate a minibatch of dialogues.
 *
 * This object is created afresh for every minibatch.
 */
class MinibatchDialogueGenerator {
    constructor(sentenceGenerator, targetLanguage, dialogueAgent, options, minibatchIdx) {
        this._sentenceGenerator = sentenceGenerator;
        this._target = targetLanguage;
        this._dialogueAgent = dialogueAgent;
        this._minibatchSize = options.minibatchSize;
        this._rng = options.rng;
        this._options = options;

        this._minibatchIdx = minibatchIdx;
        this._turnIdx = 0;

        this._debug = true;

        this._partialDialogs = new ReservoirSampler(Math.ceil(this._minibatchSize), this._rng);
        this._emptyDialogue = new PartialDialogue();
        this._maybeAddPartialDialog(this._emptyDialogue);
        this._completeDialogs = [];
        for (let turnIdx = 0; turnIdx < options.maxTurns; turnIdx++) {
            const factor = turnIdx < FACTORS.length ? FACTORS[turnIdx] : FACTORS[FACTORS.length-1];
            this._completeDialogs[turnIdx] = new ReservoirSampler(Math.ceil(this._minibatchSize * factor), this._rng);
        }
    }

    _maybeAddCompleteDialog(dlg) {
        assert(dlg.turns.length > 0);
        this._completeDialogs[dlg.turns.length-1].add(dlg.turns);
    }

    _maybeAddPartialDialog(dlg) {
        const discarded = this._partialDialogs.add(dlg);
        if (discarded !== undefined)
            this._maybeAddCompleteDialog(discarded);
        return discarded !== dlg;
    }

    _addTurn(continuations, derivation) {
        const dlg = derivation.context.dlg;

        let [agentState, userState] = derivation.value;
        let agent, agent_target, user, user_target;
        let newConstants = {};
        Object.assign(newConstants, dlg.constants);
        if (dlg.context === null) {
            // first turn in the dialogue
            assert(dlg === this._emptyDialogue);
            assert(agentState === null);
            userState = userState.optimize();
            assert(userState !== null); // not-null even after optimize
            let turnString = derivation.toString();
            turnString = turnString.replace(/ +/g, ' ');
            turnString = this._sentenceGenerator.postprocess(turnString, userState);
            mergeConstants(newConstants, turnString);

            agent = '';
            agent_target = null;
            user = turnString;
            user_target = userState.prettyprint();
        } else {
            agentState = agentState.optimize();
            userState = userState.optimize();
            assert(agentState !== null); // not-null even after optimize
            assert(userState !== null); // not-null even after optimize
            let turnString = derivation.toString();
            turnString = turnString.replace(/ +/g, ' ');
            turnString = this._sentenceGenerator.postprocess(turnString, userState);
            mergeConstants(newConstants, turnString);

            [agent, user] = turnString.split(' <sep> ');
            assert(agent);
            assert(user);

            const agentPrediction = this._target.computePrediction(dlg.context, agentState, 'agent');
            agent_target = agentPrediction.prettyprint();

            const userPrediction = this._target.computePrediction(agentState, userState, 'user');
            user_target = userPrediction.prettyprint();
        }

        continuations.put(dlg, {
            userState,
            newTurn: {
                context: dlg.context !== null ? dlg.context.prettyprint() : null,
                agent,
                agent_target,
                user,
                user_target
            },
            newConstants: {}
        });
    }

    _tryAddTurn(continuations, derivation) {
        try {
            this._addTurn(continuations, derivation);
        } catch(e) {
            console.error(derivation.toString());
            console.error(derivation.value);
            throw e;
        }
    }

    async nextTurn() {
        this._turnIdx++;
        const start = Date.now();
        if (this._debug)
            console.log(`Minibatch ${this._minibatchIdx}, turn ${this._turnIdx}`);

        const partials = this._partialDialogs.sampled;
        this._partialDialogs.reset();

        const continuations = new MultiMap;
        if (this._minibatchSize === 1) {
            assert(partials.length <= 1);
            if (partials.length > 0) {
                const derivation = this._sentenceGenerator.generateOne(partials[0]);
                if (derivation !== undefined)
                    this._tryAddTurn(continuations, derivation);
            }
        } else {
            this._sentenceGenerator.generate(partials, (depth, derivation) => {
                this._tryAddTurn(continuations, derivation);
            });
        }

        for (let i = 0; i < partials.length; i++) {
            const dlg = partials[i];
            const ourContinuations = continuations.get(dlg);

            if (ourContinuations.length === 0) {
                // if we have no continuations, mark this dialog as complete
                this._maybeAddCompleteDialog(dlg);
            } else {
                for (let { userState, newTurn, newConstants } of ourContinuations) {
                    const newDialogue = new PartialDialogue(dlg.context, dlg.turns.concat([newTurn]),
                        newConstants, dlg.execState);

                    if (this._maybeAddPartialDialog(newDialogue)) {
                        const [newContext, newExecState] = await this._dialogueAgent.execute(userState, dlg.execState);
                        newDialogue.context = newContext;
                        newDialogue.execState = newExecState;
                    }
                }
            }
        }

        const end = Date.now();
        if (this._debug) {
            console.log(`Produced ${this._partialDialogs.counter} partial dialogs this turn`);
            console.log(`Turn took ${Math.round((end-start)/1000)} seconds`);
        }
    }

    *complete() {
        for (let dlg of this._partialDialogs)
            this._maybeAddCompleteDialog(dlg);

        //assert(this._completeDialogs.length > 0);
        for (let turnIdx = 0; turnIdx < this._options.maxTurns; turnIdx++) {
            for (let dialogue of this._completeDialogs[turnIdx])
                yield dialogue;
        }
    }
}

class DialogueGenerator extends stream.Readable {
    constructor(options) {
        super({ objectMode: true });

        this._i = 0;
        this._minibatchSize = options.minibatchSize;
        this._numMinibatches = options.numMinibatches;

        this._options = options;
        this._idPrefix = options.idPrefix || '';
        this._debug = options.debug;
        this._rng = options.rng;

        options.contextual = true;
        if (options.targetPruningSize === undefined)
            options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;
        if (options.maxConstants === undefined)
            options.maxConstants = 5;
        this._target = require('../languages/' + options.targetLanguage);
        this._generator = new SentenceGenerator(options);

        this._initialized = false;
        this._dialogueAgent = this._target.createSimulator(options);
        this._minibatchIdx = 0;
    }

    async _generateMinibatch() {
        if (!this._initialized) {
            await this._generator.initialize();
            this._initialized = true;
        }

        const start = Date.now();
        let counter = 0;
        try {
            const generator = new MinibatchDialogueGenerator(this._generator, this._target, this._dialogueAgent, this._options, this._minibatchIdx++);

            for (let turn = 0; turn < this._options.maxTurns; turn++)
                await generator.nextTurn();

            for (let turns of generator.complete()) {
                const dlg = {
                    id: this._idPrefix + '' + this._i++,
                    turns
                };
                this.push(dlg);
                counter ++;
            }
        } finally {
            const end = Date.now();
            if (this._debug)
                console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds and produced ${counter} dialogues`);
        }
    }

    _read() {
        if (this._minibatchIdx >= this._numMinibatches) {
            this.push(null);
            return;
        }

        this._generateMinibatch().catch((e) => {
            this.emit('error', e);
        });
    }
}

module.exports = {
    // low-level interface
    SentenceGenerator,

    // stream interface for batch generation
    BasicSentenceGenerator,
    ContextualSentenceGenerator,
    DialogueGenerator
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of ThingTalk
4	//
5	// Copyright 2017-2018 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	54✔
13	const stream = require('stream');	54✔
14	const events = require('events');	54✔
15
16	const { parseEntity, makeDummyEntities } = require('../utils');	54✔
17
18	const $runtime = require('./runtime');	54✔
19	const importGenie = require('../genie-compiler');	54✔
20	const i18n = require('../i18n');	54✔
21	const MultiMap = require('../multimap');	54✔
22	const { ReservoirSampler, } = require('../random');	54✔
23
24	const BASIC_TARGET_GEN_SIZE = 100000;	54✔
25	const CONTEXTUAL_TARGET_GEN_SIZE = 10000;	54✔
26
27	class SentenceGenerator extends events.EventEmitter {
28	constructor(options) {
29	super();	2✔
30	this._target = require('../languages/' + options.targetLanguage);	2✔
31	this._templateFiles = options.templateFiles;	2✔
32	this._options = options;	2✔
33	this._langPack = i18n.get(options.locale);	2✔
34	this._postprocess = this._langPack.postprocessSynthetic;	2✔
35	this._grammar = null;	2✔
36	this._generator = null;	2✔
37	}
38
39	get progress() {
40	return this._grammar.progress;	2✔
41	}
42
43	postprocess(sentence, program) {
44	return this._postprocess(sentence, program, this._options.rng);	1,200,000✔
45	}
46
47	generate(contexts, callback) {
48	this._grammar.generate(contexts, callback);	2✔
49	}
50	generateOne(context) {
51	return this._grammar.generateOne(context);	×
52	}
53
54	async initialize() {
55	this._grammar = new $runtime.Grammar(this._target, this._options);	2✔
56	for (let file of this._templateFiles) {	2✔
57	const compiledTemplate = await importGenie(file);	2✔
58	this._grammar = await compiledTemplate(this._options, this._langPack, this._grammar);	2✔
59	}
60	this._grammar.on('progress', (value) => {	2✔
61	this.emit('progress', value);	4,254✔
62	});
63	}
64	}
65
66	const MINIBATCH_SIZE = 5000;	54✔
67	class ContextualSentenceGenerator extends stream.Transform {
68	constructor(options = {}) {	×
69	super({ objectMode: true });	×
70	options.contextual = true;	×
71	if (options.targetPruningSize === undefined)	×
72	options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;	×
73
74	this._options = options;	×
75	this._idPrefix = options.idPrefix;	×
76	this._debug = options.debug;	×
77	this._target = require('../languages/' + options.targetLanguage);	×
78	this._generator = new SentenceGenerator(options);	×
79
80	this._minibatch = [];	×
81	this._processed = 0;	×
82
83	this._initialized = false;	×
84	this._i = 0;	×
85	}
86
87	_output(depth, derivation) {
88	const context = derivation.context;	×
89	const program = derivation.value.optimize();	×
90	assert(program !== null); // not-null even after optimize	×
91	let preprocessed = derivation.toString();	×
92	preprocessed = preprocessed.replace(/ +/g, ' ');	×
93	preprocessed = this._generator.postprocess(preprocessed, program);	×
94
95	const [contextCode, contextEntities] = this._target.serializeNormalized(context.value);	×
96	let sequence;
97	try {	×
98	sequence = this._target.serialize(program, contextCode, contextEntities);	×
99	} catch(e) {
100	//console.error(context.code.join(' '));
101	console.error(preprocessed);	×
102	console.error(program.prettyprint());	×
103	console.error(sequence);	×
104
105	console.error(program.prettyprint(program).trim());	×
106	this.emit('error', e);	×
107	}
108
109	let id = String(this._i++);	×
110	id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;	×
111	const flags = {	×
112	synthetic: true,
113	contextual: true,
114	};
115
116	this.push({ depth, id, flags, preprocessed, context: contextCode.join(' '), target_code: sequence.join(' ') });	×
117	}
118
119	async _process(minibatch) {
120	if (!this._initialized) {	×
121	await this._generator.initialize();	×
122	this._initialized = true;	×
123	}
124
125	const start = Date.now();	×
126	if (this._debug)	×
127	console.log(`Minibatch ${this._processed}-${this._processed+minibatch.length}`);	×
128
129	const contexts = await Promise.all(minibatch.map(async (contextCode) => {	×
130	const code = contextCode.split(' ');	×
131
132	const entities = makeDummyEntities(contextCode);	×
133	const program = await this._target.parse(code, entities, this._options);	×
134	return new $runtime.Context(code, program, entities);	×
135	}));
136
137	this._generator.generate(contexts, this._output.bind(this));	×
138
139	this._processed += minibatch.length;	×
140	const end = Date.now();	×
141	if (this._debug)	×
142	console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds`);	×
143	}
144
145	_transform(contextExample, encoding, callback) {
146	this._minibatch.push(contextExample);	×
147	if (this._minibatch.length < MINIBATCH_SIZE) {	×
148	callback();	×
149	return;	×
150	}
151
152	const minibatch = this._minibatch;	×
153	this._minibatch = [];	×
154	this._process(minibatch).then(callback, callback);	×
155	}
156
157	_flush(callback) {
158	if (this._minibatch.length > 0)	×
159	this._process(this._minibatch).then(callback, callback);	×
160	else
161	process.nextTick(callback);	×
162	}
163	}
164
165	class BasicSentenceGenerator extends stream.Readable {
166	constructor(options = {}) {	×
167	super({ objectMode: true });	2✔
168	options.contextual = false;	2✔
169	if (options.targetPruningSize === undefined)	2!
170	options.targetPruningSize = BASIC_TARGET_GEN_SIZE;	×
171	this._target = require('../languages/' + options.targetLanguage);	2✔
172	this._idPrefix = options.idPrefix \|\| '';	2✔
173	this._generator = new SentenceGenerator(options);	2✔
174	this._generator.on('progress', (value) => {	2✔
175	this.emit('progress', value);	4,254✔
176	});
177	this._iterator = null;	2✔
178
179	this._initialization = null;	2✔
180	this._i = 0;	2✔
181	}
182
183	_read() {
184	if (this._initialization === null)	2!
185	this._initialization = this._generator.initialize();	2✔
186
187	this._initialization.then(() => {	2✔
188	this._generator.generate([], this._output.bind(this));	2✔
189	this.emit('progress', this._generator.progress);	2✔
190	this.push(null);	2✔
191	}).catch((e) => {
192	console.error(e);	×
193	this.emit('error', e);	×
194	});
195	}
196
197	_output(depth, derivation) {
198	let program = derivation.value.optimize();	1,200,000✔
199	assert(program !== null); // not-null even after optimize	1,200,000✔
200	let preprocessed = derivation.toString();	1,200,000✔
201	preprocessed = preprocessed.replace(/ +/g, ' ');	1,200,000✔
202	preprocessed = this._generator.postprocess(preprocessed, program);	1,200,000✔
203	let sequence;
204	try {	1,200,000✔
205	sequence = this._target.serialize(program, [], {});	1,200,000✔
206	//ThingTalk.NNSyntax.fromNN(sequence, {});
207
208	if (sequence.some((t) => t.endsWith(':undefined')))	8,489,740!
209	throw new TypeError(`Generated undefined type`);	×
210	} catch(e) {
211	console.error(preprocessed);	×
212	console.error(String(program));	×
213	console.error(sequence);	×
214
215	console.error(program.prettyprint().trim());	×
216	this.emit('error', e);	×
217	}
218	let id = String(this._i++);	1,200,000✔
219	id = this._idPrefix + depth + '000000000'.substring(0,9-id.length) + id;	1,200,000✔
220	const flags = {	1,200,000✔
221	synthetic: true
222	};
223	return this.push({ depth, id, flags, preprocessed, target_code: sequence.join(' ') });	1,200,000✔
224	}
225	}
226
227	class PartialDialogue {
228	constructor(context = null, turns = [], constants = {}, execState = undefined) {	×
229	this.context = context;	×
230	this.turns = turns;	×
231	this.constants = constants;	×
232	this.execState = undefined;	×
233	}
234	}
235
236	function mergeConstants(constants, sentence) {
237	for (let token of sentence.split(' ')) {	×
238	const entity = parseEntity(token);	×
239	if (entity === null)	×
240	continue;	×
241
242	const [type, index] = entity;	×
243	constants[type] = Math.max(constants[type] \|\| 0, index + 1);	×
244	}
245	}
246
247	const FACTORS = [50, 75, 75, 100];	54✔
248
249	/**
250	* Generate a minibatch of dialogues.
251	*
252	* This object is created afresh for every minibatch.
253	*/
254	class MinibatchDialogueGenerator {
255	constructor(sentenceGenerator, targetLanguage, dialogueAgent, options, minibatchIdx) {
256	this._sentenceGenerator = sentenceGenerator;	×
257	this._target = targetLanguage;	×
258	this._dialogueAgent = dialogueAgent;	×
259	this._minibatchSize = options.minibatchSize;	×
260	this._rng = options.rng;	×
261	this._options = options;	×
262
263	this._minibatchIdx = minibatchIdx;	×
264	this._turnIdx = 0;	×
265
266	this._debug = true;	×
267
NEW 268	this._partialDialogs = new ReservoirSampler(Math.ceil(this._minibatchSize), this._rng);	×
269	this._emptyDialogue = new PartialDialogue();	×
270	this._maybeAddPartialDialog(this._emptyDialogue);	×
NEW 271	this._completeDialogs = [];	×
NEW 272	for (let turnIdx = 0; turnIdx < options.maxTurns; turnIdx++) {	×
NEW 273	const factor = turnIdx < FACTORS.length ? FACTORS[turnIdx] : FACTORS[FACTORS.length-1];	×
NEW 274	this._completeDialogs[turnIdx] = new ReservoirSampler(Math.ceil(this._minibatchSize * factor), this._rng);	×
275	}
276	}
277
278	_maybeAddCompleteDialog(dlg) {
279	assert(dlg.turns.length > 0);	×
NEW 280	this._completeDialogs[dlg.turns.length-1].add(dlg.turns);	×
281	}
282
283	_maybeAddPartialDialog(dlg) {
284	const discarded = this._partialDialogs.add(dlg);	×
285	if (discarded !== undefined)	×
286	this._maybeAddCompleteDialog(discarded);	×
287	return discarded !== dlg;	×
288	}
289
290	_addTurn(continuations, derivation) {
291	const dlg = derivation.context.dlg;	×
292
293	let [agentState, userState] = derivation.value;	×
294	let agent, agent_target, user, user_target;
295	let newConstants = {};	×
296	Object.assign(newConstants, dlg.constants);	×
297	if (dlg.context === null) {	×
298	// first turn in the dialogue
299	assert(dlg === this._emptyDialogue);	×
300	assert(agentState === null);	×
301	userState = userState.optimize();	×
302	assert(userState !== null); // not-null even after optimize	×
303	let turnString = derivation.toString();	×
304	turnString = turnString.replace(/ +/g, ' ');	×
305	turnString = this._sentenceGenerator.postprocess(turnString, userState);	×
306	mergeConstants(newConstants, turnString);	×
307
308	agent = '';	×
309	agent_target = null;	×
310	user = turnString;	×
311	user_target = userState.prettyprint();	×
312	} else {
313	agentState = agentState.optimize();	×
314	userState = userState.optimize();	×
315	assert(agentState !== null); // not-null even after optimize	×
316	assert(userState !== null); // not-null even after optimize	×
317	let turnString = derivation.toString();	×
318	turnString = turnString.replace(/ +/g, ' ');	×
319	turnString = this._sentenceGenerator.postprocess(turnString, userState);	×
320	mergeConstants(newConstants, turnString);	×
321
322	[agent, user] = turnString.split(' <sep> ');	×
323	assert(agent);	×
324	assert(user);	×
325
326	const agentPrediction = this._target.computePrediction(dlg.context, agentState, 'agent');	×
327	agent_target = agentPrediction.prettyprint();	×
328
329	const userPrediction = this._target.computePrediction(agentState, userState, 'user');	×
330	user_target = userPrediction.prettyprint();	×
331	}
332
333	continuations.put(dlg, {	×
334	userState,
335	newTurn: {
336	context: dlg.context !== null ? dlg.context.prettyprint() : null,	×
337	agent,
338	agent_target,
339	user,
340	user_target
341	},
342	newConstants: {}
343	});
344	}
345
346	_tryAddTurn(continuations, derivation) {
347	try {	×
348	this._addTurn(continuations, derivation);	×
349	} catch(e) {
350	console.error(derivation.toString());	×
351	console.error(derivation.value);	×
352	throw e;	×
353	}
354	}
355
356	async nextTurn() {
357	this._turnIdx++;	×
358	const start = Date.now();	×
359	if (this._debug)	×
360	console.log(`Minibatch ${this._minibatchIdx}, turn ${this._turnIdx}`);	×
361
362	const partials = this._partialDialogs.sampled;	×
363	this._partialDialogs.reset();	×
364
365	const continuations = new MultiMap;	×
366	if (this._minibatchSize === 1) {	×
367	assert(partials.length <= 1);	×
368	if (partials.length > 0) {	×
369	const derivation = this._sentenceGenerator.generateOne(partials[0]);	×
370	if (derivation !== undefined)	×
371	this._tryAddTurn(continuations, derivation);	×
372	}
373	} else {
374	this._sentenceGenerator.generate(partials, (depth, derivation) => {	×
375	this._tryAddTurn(continuations, derivation);	×
376	});
377	}
378
379	for (let i = 0; i < partials.length; i++) {	×
380	const dlg = partials[i];	×
381	const ourContinuations = continuations.get(dlg);	×
382
383	if (ourContinuations.length === 0) {	×
384	// if we have no continuations, mark this dialog as complete
385	this._maybeAddCompleteDialog(dlg);	×
386	} else {
387	for (let { userState, newTurn, newConstants } of ourContinuations) {	×
388	const newDialogue = new PartialDialogue(dlg.context, dlg.turns.concat([newTurn]),	×
389	newConstants, dlg.execState);
390
391	if (this._maybeAddPartialDialog(newDialogue)) {	×
392	const [newContext, newExecState] = await this._dialogueAgent.execute(userState, dlg.execState);	×
393	newDialogue.context = newContext;	×
394	newDialogue.execState = newExecState;	×
395	}
396	}
397	}
398	}
399
400	const end = Date.now();	×
401	if (this._debug) {	×
402	console.log(`Produced ${this._partialDialogs.counter} partial dialogs this turn`);	×
403	console.log(`Turn took ${Math.round((end-start)/1000)} seconds`);	×
404	}
405	}
406
407	*complete() {
408	for (let dlg of this._partialDialogs)	×
409	this._maybeAddCompleteDialog(dlg);	×
410
411	//assert(this._completeDialogs.length > 0);
NEW 412	for (let turnIdx = 0; turnIdx < this._options.maxTurns; turnIdx++) {	×
NEW 413	for (let dialogue of this._completeDialogs[turnIdx])	×
NEW 414	yield dialogue;	×
415	}
416	}
417	}
418
419	class DialogueGenerator extends stream.Readable {
420	constructor(options) {
421	super({ objectMode: true });	×
422
423	this._i = 0;	×
424	this._minibatchSize = options.minibatchSize;	×
NEW 425	this._numMinibatches = options.numMinibatches;	×
426
427	this._options = options;	×
428	this._idPrefix = options.idPrefix \|\| '';	×
429	this._debug = options.debug;	×
430	this._rng = options.rng;	×
431
432	options.contextual = true;	×
433	if (options.targetPruningSize === undefined)	×
434	options.targetPruningSize = CONTEXTUAL_TARGET_GEN_SIZE;	×
435	if (options.maxConstants === undefined)	×
436	options.maxConstants = 5;	×
437	this._target = require('../languages/' + options.targetLanguage);	×
438	this._generator = new SentenceGenerator(options);	×
439
440	this._initialized = false;	×
441	this._dialogueAgent = this._target.createSimulator(options);	×
442	this._minibatchIdx = 0;	×
443	}
444
445	async _generateMinibatch() {
446	if (!this._initialized) {	×
447	await this._generator.initialize();	×
448	this._initialized = true;	×
449	}
450
451	const start = Date.now();	×
452	let counter = 0;	×
453	try {	×
454	const generator = new MinibatchDialogueGenerator(this._generator, this._target, this._dialogueAgent, this._options, this._minibatchIdx++);	×
455
456	for (let turn = 0; turn < this._options.maxTurns; turn++)	×
457	await generator.nextTurn();	×
458
NEW 459	for (let turns of generator.complete()) {	×
460	const dlg = {	×
461	id: this._idPrefix + '' + this._i++,
462	turns
463	};
464	this.push(dlg);	×
465	counter ++;	×
466	}
467	} finally {
468	const end = Date.now();	×
469	if (this._debug)	×
470	console.log(`Minibatch took ${Math.round((end-start)/1000)} seconds and produced ${counter} dialogues`);	×
471	}
472	}
473
474	_read() {
NEW 475	if (this._minibatchIdx >= this._numMinibatches) {	×
476	this.push(null);	×
477	return;	×
478	}
479
480	this._generateMinibatch().catch((e) => {	×
481	this.emit('error', e);	×
482	});
483	}
484	}
485
486	module.exports = {	54✔
487	// low-level interface
488	SentenceGenerator,
489
490	// stream interface for batch generation
491	BasicSentenceGenerator,
492	ContextualSentenceGenerator,
493	DialogueGenerator
494	};

stanford-oval / genie-toolkit / 835

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous