stanford-oval / genie-toolkit / 641

pending completion

Build # 641

Build Type

Pull #137

travis-ci-com

Committed by

web-flow

Commit Message

Add templates for basic ThingTalk

These templates correspond exactly to the kind of natural language
we exact match on - no more, no less. They are designed so we can
expand them at very high depth and very high target pruning size,
without fear of bad run time.

Pull Request Pull Request #137: Second round of dialogue templates

Run Details

1727 of 4532 branches covered (38.11%)

Branch coverage included in aggregate %.

17 of 71 new or added lines in 7 files covered. (23.94%)

38 existing lines in 5 files now uncovered.

4346 of 9197 relevant lines covered (47.25%)

583529.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.47

/tool/dialog-to-contextual.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const Tp = require('thingpedia');
const Stream = require('stream');
const fs = require('fs');

const { AVAILABLE_LANGUAGES } = require('../lib/languages');
const TokenizerService = require('../lib/tokenizer');
const { DatasetStringifier } = require('../lib/dataset-parsers');
const StreamUtils = require('../lib/stream-utils');
const Utils = require('../lib/utils');

const { DialogueParser } = require('./lib/dialog_parser');
const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');

class DialogueToTurnStream extends Stream.Transform {
    constructor(options) {
        super({ objectMode: true });

        this._locale = options.locale;

        // FIXME should not load ThingTalk here
        this._options = options;
        this._debug = options.debug;
        this._side = options.side;
        this._flags = options.flags;
        this._target = require('../lib/languages/' + options.targetLanguage);

        this._tokenized = options.tokenized;
        this._tokenizer = null;
        if (!this._tokenized)
            this._tokenizer = TokenizerService.get('local', true);
    }

    async _preprocess(sentence, contextEntities) {
        let tokenized;
        if (this._tokenized) {
            const tokens = sentence.split(' ');
            const entities = Utils.makeDummyEntities(sentence);
            tokenized = { tokens, entities };
        } else {
            tokenized = await this._tokenizer.tokenize(this._locale, sentence);
        }
        Utils.renumberEntities(tokenized, contextEntities);
        return tokenized;
    }

    async _emitAgentTurn(i, turn, dlg) {
        if (i === 0)
            return;

        const context = await this._target.parse(turn.context, this._options);
        const agentContext = this._target.prepareContextForPrediction(context, 'agent');
        const [contextCode, contextEntities] = this._target.serializeNormalized(agentContext);

        const agentTarget = await this._target.parse(turn.agent_target, this._options);
        const agentCode = await this._target.serializePrediction(agentTarget, '', contextEntities, 'agent');

        const { tokens, } = await this._preprocess(turn.agent, contextEntities);

        this.push({
            id: this._flags + '' + dlg.id + '/' + i,
            context: contextCode.join(' '),
            preprocessed: tokens.join(' '),
            target_code: agentCode.join(' ')
        });
    }

    async _emitUserTurn(i, turn, dlg) {
        let context, contextCode, contextEntities;
        if (i > 0) {
            context = await this._target.parse(turn.context, this._options);
            // apply the agent prediction to the context to get the state of the dialogue before
            // the user speaks
            const agentPrediction = await this._target.parse(turn.agent_target, this._options);
            context = this._target.computeNewState(context, agentPrediction);

            const userContext = this._target.prepareContextForPrediction(context, 'user');
            [contextCode, contextEntities] = this._target.serializeNormalized(userContext);
        } else {
            context = null;
            contextCode = ['null'];
            contextEntities = {};
        }

        const { tokens, entities } = await this._preprocess(turn.user, contextEntities);
        const userTarget = await this._target.parse(turn.user_target, this._options);
        const code = await this._target.serializePrediction(userTarget, tokens, entities, 'user');

        this.push({
            id: this._flags + '' + dlg.id + '/' + i,
            context: contextCode.join(' '),
            preprocessed: tokens.join(' '),
            target_code: code.join(' ')
        });
    }

    async _doTransform(dlg) {
        for (let i = 0; i < dlg.length; i++) {
            const turn = dlg[i];

            try {
                if (this._side === 'agent')
                    await this._emitAgentTurn(i, turn, dlg);
                else
                    await this._emitUserTurn(i, turn, dlg);
            } catch(e) {
                console.error(turn);
                throw e;
            }
        }

    }

    _transform(dialog, encoding, callback) {
        this._doTransform(dialog).then(() => callback(null), callback);
    }

    _flush(callback) {
        if (this._tokenizer)
            this._tokenizer.end();
        process.nextTick(callback);
    }
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('dialog-to-contextual', {
            addHelp: true,
            description: "Transform a dialog input file into a contextual dataset (turn by turn)."
        });
        parser.addArgument(['-o', '--output'], {
            required: true,
            type: fs.createWriteStream
        });
        parser.addArgument('--tokenized', {
            required: false,
            action: 'storeTrue',
            defaultValue: true,
            help: "The dataset is already tokenized (this is the default)."
        });
        parser.addArgument('--no-tokenized', {
            required: false,
            dest: 'tokenized',
            action: 'storeFalse',
            help: "The dataset is not already tokenized."
        });
        parser.addArgument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.addArgument(['-t', '--target-language'], {
            required: false,
            defaultValue: 'thingtalk',
            choices: AVAILABLE_LANGUAGES,
            help: `The programming language to generate`
        });
        parser.addArgument('--side', {
            required: true,
            choices: ['user', 'agent'],
            help: 'Which side of the conversation should be extracted.'
        });
        parser.addArgument('--flags', {
            required: false,
            defaultValue: '',
            help: 'Additional flags to add to the generated training examples.'
        });
        parser.addArgument('input_file', {
            nargs: '+',
            type: maybeCreateReadStream,
            help: 'Input dialog file; use - for standard input'
        });
        parser.addArgument(['-l', '--locale'], {
            required: false,
            defaultValue: 'en-US',
            help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
        });
        parser.addArgument('--debug', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Enable debugging.',
            defaultValue: true
        });
        parser.addArgument('--no-debug', {
            nargs: 0,
            action: 'storeFalse',
            dest: 'debug',
            help: 'Disable debugging.',
        });
    },

    async execute(args) {
        let tpClient = null;
        if (args.thingpedia)
            tpClient = new Tp.FileClient(args);

        readAllLines(args.input_file, '====')
            .pipe(new DialogueParser())
            .pipe(new DialogueToTurnStream({
                locale: args.locale,
                targetLanguage: args.target_language,
                thingpediaClient: tpClient,
                flags: args.flags,
                side: args.side,
                tokenized: args.tokenized,
                debug: args.debug
            }))
            .pipe(new DatasetStringifier())
            .pipe(args.output);

        await StreamUtils.waitFinish(args.output);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const Tp = require('thingpedia');	21✔
13	const Stream = require('stream');	21✔
14	const fs = require('fs');	21✔
15
16	const { AVAILABLE_LANGUAGES } = require('../lib/languages');	21✔
17	const TokenizerService = require('../lib/tokenizer');	21✔
18	const { DatasetStringifier } = require('../lib/dataset-parsers');	21✔
19	const StreamUtils = require('../lib/stream-utils');	21✔
20	const Utils = require('../lib/utils');	21✔
21
22	const { DialogueParser } = require('./lib/dialog_parser');	21✔
23	const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');	21✔
24
25	class DialogueToTurnStream extends Stream.Transform {
26	constructor(options) {
27	super({ objectMode: true });	×
28
29	this._locale = options.locale;	×
30
31	// FIXME should not load ThingTalk here
32	this._options = options;	×
33	this._debug = options.debug;	×
34	this._side = options.side;	×
35	this._flags = options.flags;	×
36	this._target = require('../lib/languages/' + options.targetLanguage);	×
37
38	this._tokenized = options.tokenized;	×
39	this._tokenizer = null;	×
40	if (!this._tokenized)	×
41	this._tokenizer = TokenizerService.get('local', true);	×
42	}
43
44	async _preprocess(sentence, contextEntities) {
45	let tokenized;
46	if (this._tokenized) {	×
47	const tokens = sentence.split(' ');	×
48	const entities = Utils.makeDummyEntities(sentence);	×
49	tokenized = { tokens, entities };	×
50	} else {
51	tokenized = await this._tokenizer.tokenize(this._locale, sentence);	×
52	}
53	Utils.renumberEntities(tokenized, contextEntities);	×
54	return tokenized;	×
55	}
56
57	async _emitAgentTurn(i, turn, dlg) {
58	if (i === 0)	×
59	return;	×
60
61	const context = await this._target.parse(turn.context, this._options);	×
62	const agentContext = this._target.prepareContextForPrediction(context, 'agent');	×
63	const [contextCode, contextEntities] = this._target.serializeNormalized(agentContext);	×
64
65	const agentTarget = await this._target.parse(turn.agent_target, this._options);	×
NEW 66	const agentCode = await this._target.serializePrediction(agentTarget, '', contextEntities, 'agent');	×
67
68	const { tokens, } = await this._preprocess(turn.agent, contextEntities);	×
69
70	this.push({	×
71	id: this._flags + '' + dlg.id + '/' + i,
72	context: contextCode.join(' '),
73	preprocessed: tokens.join(' '),
74	target_code: agentCode.join(' ')
75	});
76	}
77
78	async _emitUserTurn(i, turn, dlg) {
79	let context, contextCode, contextEntities;
80	if (i > 0) {	×
NEW 81	context = await this._target.parse(turn.context, this._options);	×
82	// apply the agent prediction to the context to get the state of the dialogue before
83	// the user speaks
NEW 84	const agentPrediction = await this._target.parse(turn.agent_target, this._options);	×
NEW 85	context = this._target.computeNewState(context, agentPrediction);	×
86
87	const userContext = this._target.prepareContextForPrediction(context, 'user');	×
88	[contextCode, contextEntities] = this._target.serializeNormalized(userContext);	×
89	} else {
90	context = null;	×
91	contextCode = ['null'];	×
92	contextEntities = {};	×
93	}
94
95	const { tokens, entities } = await this._preprocess(turn.user, contextEntities);	×
96	const userTarget = await this._target.parse(turn.user_target, this._options);	×
NEW 97	const code = await this._target.serializePrediction(userTarget, tokens, entities, 'user');	×
98
99	this.push({	×
100	id: this._flags + '' + dlg.id + '/' + i,
101	context: contextCode.join(' '),
102	preprocessed: tokens.join(' '),
103	target_code: code.join(' ')
104	});
105	}
106
107	async _doTransform(dlg) {
108	for (let i = 0; i < dlg.length; i++) {	×
109	const turn = dlg[i];	×
110
111	try {	×
112	if (this._side === 'agent')	×
113	await this._emitAgentTurn(i, turn, dlg);	×
114	else
115	await this._emitUserTurn(i, turn, dlg);	×
116	} catch(e) {
117	console.error(turn);	×
118	throw e;	×
119	}
120	}
121
122	}
123
124	_transform(dialog, encoding, callback) {
125	this._doTransform(dialog).then(() => callback(null), callback);	×
126	}
127
128	_flush(callback) {
129	if (this._tokenizer)	×
130	this._tokenizer.end();	×
131	process.nextTick(callback);	×
132	}
133	}
134
135	module.exports = {	21✔
136	initArgparse(subparsers) {
137	const parser = subparsers.addParser('dialog-to-contextual', {	21✔
138	addHelp: true,
139	description: "Transform a dialog input file into a contextual dataset (turn by turn)."
140	});
141	parser.addArgument(['-o', '--output'], {	21✔
142	required: true,
143	type: fs.createWriteStream
144	});
145	parser.addArgument('--tokenized', {	21✔
146	required: false,
147	action: 'storeTrue',
148	defaultValue: true,
149	help: "The dataset is already tokenized (this is the default)."
150	});
151	parser.addArgument('--no-tokenized', {	21✔
152	required: false,
153	dest: 'tokenized',
154	action: 'storeFalse',
155	help: "The dataset is not already tokenized."
156	});
157	parser.addArgument('--thingpedia', {	21✔
158	required: true,
159	help: 'Path to ThingTalk file containing class definitions.'
160	});
161	parser.addArgument(['-t', '--target-language'], {	21✔
162	required: false,
163	defaultValue: 'thingtalk',
164	choices: AVAILABLE_LANGUAGES,
165	help: `The programming language to generate`
166	});
167	parser.addArgument('--side', {	21✔
168	required: true,
169	choices: ['user', 'agent'],
170	help: 'Which side of the conversation should be extracted.'
171	});
172	parser.addArgument('--flags', {	21✔
173	required: false,
174	defaultValue: '',
175	help: 'Additional flags to add to the generated training examples.'
176	});
177	parser.addArgument('input_file', {	21✔
178	nargs: '+',
179	type: maybeCreateReadStream,
180	help: 'Input dialog file; use - for standard input'
181	});
182	parser.addArgument(['-l', '--locale'], {	21✔
183	required: false,
184	defaultValue: 'en-US',
185	help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
186	});
187	parser.addArgument('--debug', {	21✔
188	nargs: 0,
189	action: 'storeTrue',
190	help: 'Enable debugging.',
191	defaultValue: true
192	});
193	parser.addArgument('--no-debug', {	21✔
194	nargs: 0,
195	action: 'storeFalse',
196	dest: 'debug',
197	help: 'Disable debugging.',
198	});
199	},
200
201	async execute(args) {
202	let tpClient = null;	×
203	if (args.thingpedia)	×
204	tpClient = new Tp.FileClient(args);	×
205
206	readAllLines(args.input_file, '====')	×
207	.pipe(new DialogueParser())
208	.pipe(new DialogueToTurnStream({
209	locale: args.locale,
210	targetLanguage: args.target_language,
211	thingpediaClient: tpClient,
212	flags: args.flags,
213	side: args.side,
214	tokenized: args.tokenized,
215	debug: args.debug
216	}))
217	.pipe(new DatasetStringifier())
218	.pipe(args.output);
219
220	await StreamUtils.waitFinish(args.output);	×
221	}
222	};

stanford-oval / genie-toolkit / 641

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous