1987

pending completion

Build # 1987

Build Type

Pull #618

travis-ci-com

Committed by

web-flow

Commit Message

Merge 02ab9d225 into b74f3c67e

Pull Request Pull Request #618: Fix evaluation of Thingpedia model

Run Details

12283 of 25943 branches covered (47.35%)

Branch coverage included in aggregate %.

19 of 29 new or added lines in 4 files covered. (65.52%)

1 existing line in 1 file now uncovered.

28566 of 48731 relevant lines covered (58.62%)

445888.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

25.0

/tool/extract-predicted-slots.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as argparse from 'argparse';
import * as Tp from 'thingpedia';
import * as Stream from 'stream';
import assert from 'assert';
import * as fs from 'fs';
import JSONStream from 'JSONStream';
import { Ast, SchemaRetriever, } from 'thingtalk';

import * as StreamUtils from '../lib/utils/stream-utils';
import * as Utils from '../lib/utils/misc-utils';
import { EntityMap } from '../lib/utils/entity-utils';
import * as I18n from '../lib/i18n';
import * as ThingTalkUtils from '../lib/utils/thingtalk';
import {
    DialogueParser,
    ParsedDialogue,
    DialogueTurn
} from '../lib/dataset-tools/parsers';
import SlotExtractor from '../lib/dataset-tools/evaluation/slot_extractor';
import { SimulationDatabase } from '../lib/dialogue-agent/simulator/types';

import { maybeCreateReadStream, readAllLines } from './lib/argutils';
import MultiJSONDatabase from './lib/multi_json_database';
import * as ParserClient from '../lib/prediction/parserclient';

interface DialogueToDSTStreamOptions {
    locale : string;
    thingpediaClient : Tp.BaseClient;
    schemaRetriever ?: SchemaRetriever;
    database : SimulationDatabase|undefined;
    tokenized : boolean;
    parser : ParserClient.ParserClient;
    debug : boolean;
}

class DialogueToDSTStream extends Stream.Transform {
    private _slotExtractor : SlotExtractor;
    private _tokenized : boolean;
    private _tokenizer : I18n.BaseTokenizer;
    private _parser : ParserClient.ParserClient;
    private _options : DialogueToDSTStreamOptions;

    constructor(options : DialogueToDSTStreamOptions) {
        super({ objectMode: true });

        if (!options.schemaRetriever)
            options.schemaRetriever = new SchemaRetriever(options.thingpediaClient, null, true);
        this._slotExtractor = new SlotExtractor(options.locale, options.thingpediaClient, options.schemaRetriever, options.database);
        this._tokenized = options.tokenized;
        this._tokenizer = I18n.get(options.locale).getTokenizer();
        this._parser = options.parser;
        this._options = options;
    }

    private _preprocess(sentence : string, contextEntities : EntityMap) {
        let tokenized;
        if (this._tokenized) {
            const tokens = sentence.split(' ');
            const entities = Utils.makeDummyEntities(sentence);
            tokenized = { tokens, entities };
        } else {
            tokenized = this._tokenizer.tokenize(sentence);
        }
        Utils.renumberEntities(tokenized, contextEntities);
        return tokenized;
    }

    private async _checkTurn(id : string, turn : DialogueTurn, turnIndex : number) : Promise<[Record<string, string>, Record<string, string>]> {
        let context, contextCode, contextEntities;
        if (turnIndex > 0) {
            if (turn.intermediate_context) {
                context = await ThingTalkUtils.parse(turn.intermediate_context, this._options);
                assert(context instanceof Ast.DialogueState);
            } else {
                context = await ThingTalkUtils.parse(turn.context!, this._options);
                assert(context instanceof Ast.DialogueState);
                // apply the agent prediction to the context to get the state of the dialogue before
                // the user speaks
                const agentPrediction = await ThingTalkUtils.parse(turn.agent_target!, this._options);
                assert(agentPrediction instanceof Ast.DialogueState);
                context = ThingTalkUtils.computeNewState(context, agentPrediction, 'agent');
            }

            const userContext = ThingTalkUtils.prepareContextForPrediction(context, 'user');
            [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(userContext);
        } else {
            context = null;
            contextCode = ['null'];
            contextEntities = {};
        }

        const { tokens, entities } = this._preprocess(turn.user, contextEntities);
        const goldUserTarget = await ThingTalkUtils.parse(turn.user_target, this._options);
        assert(goldUserTarget instanceof Ast.DialogueState);
        const goldUserState = ThingTalkUtils.computeNewState(context, goldUserTarget, 'user');
        const goldSlots = await this._slotExtractor.extractSlots(goldUserState);

        const parsed = await this._parser.sendUtterance(tokens.join(' '), contextCode, contextEntities, {
            tokenized: true,
            skip_typechecking: true
        });

        const predictions = parsed.candidates
            .filter((beam) => beam.score !== 'Infinity') // ignore exact matches
            .map((beam) => beam.code);

        if (predictions.length === 0)
            return [goldSlots, {}];

        const choice = predictions[0];

        let predictedUserTarget;
        try {
            predictedUserTarget = await ThingTalkUtils.parsePrediction(choice, entities, this._options);
        } catch(e) {
            return [goldSlots, {}];
        }
        if (predictedUserTarget === null)
            return [goldSlots, {}];

        assert(predictedUserTarget instanceof Ast.DialogueState);
        const predictedUserState = ThingTalkUtils.computeNewState(context, predictedUserTarget, 'user');
        let predictedSlots;
        try {
            predictedSlots = await this._slotExtractor.extractSlots(predictedUserState);
        } catch(e) {
            console.error(predictedUserTarget.prettyprint());
            throw e;
        }

        return [goldSlots, predictedSlots];
    }

    private async _doDialogue(dlg : ParsedDialogue) : Promise<[string, Record<number, { turn_belief : Record<string, string>, pred_bs_ptr : Record<string, string> }>]> {
        const output : Record<number, { turn_belief : Record<string, string>, pred_bs_ptr : Record<string, string> }> = [];
        for (let i = 0; i < dlg.length; i++) {
            const turn = dlg[i];

            const [goldSlots, predictedSlots] = await this._checkTurn(dlg.id, turn, i);
            output[i] = {
                turn_belief: goldSlots,
                pred_bs_ptr: predictedSlots
            };
        }

        return [dlg.id, output];
    }

    _transform(dlg : ParsedDialogue, encoding : BufferEncoding, callback : (error : Error|null, result ?: unknown) => void) {
        this._doDialogue(dlg).then((result) => callback(null, result), callback);
    }

    _flush(callback : () => void) {
        process.nextTick(callback);
    }
}

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('extract-predicted-slots', {
        add_help: true,
        description: "Transform a dialog input file in ThingTalk format into a dialogue state tracking prediction file."
    });
    parser.add_argument('-o', '--output', {
        required: false,
        default: process.stdout,
        type: fs.createWriteStream,
        help: "Write results to this file instead of stdout"
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
    });
    parser.add_argument('--url', {
        required: false,
        help: "URL of the server to evaluate. Use a file:// URL pointing to a model directory to evaluate using a local instance of genienlp",
        default: 'http://127.0.0.1:8400',
    });
    parser.add_argument('--tokenized', {
        required: false,
        action: 'store_true',
        default: false,
        help: "The dataset is already tokenized."
    });
    parser.add_argument('--no-tokenized', {
        required: false,
        dest: 'tokenized',
        action: 'store_false',
        help: "The dataset is not already tokenized (this is the default)."
    });
    parser.add_argument('--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('input_file', {
        nargs: '+',
        type: maybeCreateReadStream,
        help: 'Input datasets to evaluate (in dialog format); use - for standard input'
    });
    parser.add_argument('--debug', {
        action: 'store_true',
        help: 'Enable debugging.',
        default: true
    });
    parser.add_argument('--no-debug', {
        action: 'store_false',
        dest: 'debug',
        help: 'Disable debugging.',
    });
    parser.add_argument('--database-file', {
        required: false,
        help: `Path to a file pointing to JSON databases used to simulate queries.`,
    });
}

export async function execute(args : any) {
    const tpClient = new Tp.FileClient(args);
    const parser = ParserClient.get(args.url, args.locale);
    await parser.start();

    let database;
    if (args.database_file) {
        database = new MultiJSONDatabase(args.database_file);
        await database.load();
    }

    readAllLines(args.input_file, '====')
        .pipe(new DialogueParser())
        .pipe(new DialogueToDSTStream({
            locale: args.locale,
            debug: args.debug,
            tokenized: args.tokenized,
            thingpediaClient: tpClient,
            database: database,
            parser: parser,
        }))
        .pipe((JSONStream as any /* FIXME bad type declaration */).stringifyObject(undefined, undefined, undefined, 2))
        .pipe(args.output);

    await StreamUtils.waitFinish(args.output);

    await parser.stop();
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as argparse from 'argparse';
22	import * as Tp from 'thingpedia';	90✔
23	import * as Stream from 'stream';	90✔
24	import assert from 'assert';	90✔
25	import * as fs from 'fs';	90✔
26	import JSONStream from 'JSONStream';	90✔
27	import { Ast, SchemaRetriever, } from 'thingtalk';	90✔
28
29	import * as StreamUtils from '../lib/utils/stream-utils';	90✔
30	import * as Utils from '../lib/utils/misc-utils';	90✔
31	import { EntityMap } from '../lib/utils/entity-utils';
32	import * as I18n from '../lib/i18n';	90✔
33	import * as ThingTalkUtils from '../lib/utils/thingtalk';	90✔
34	import {	90✔
35	DialogueParser,
36	ParsedDialogue,
37	DialogueTurn
38	} from '../lib/dataset-tools/parsers';
39	import SlotExtractor from '../lib/dataset-tools/evaluation/slot_extractor';	90✔
40	import { SimulationDatabase } from '../lib/dialogue-agent/simulator/types';
41
42	import { maybeCreateReadStream, readAllLines } from './lib/argutils';	90✔
43	import MultiJSONDatabase from './lib/multi_json_database';	90✔
44	import * as ParserClient from '../lib/prediction/parserclient';	90✔
45
46	interface DialogueToDSTStreamOptions {
47	locale : string;
48	thingpediaClient : Tp.BaseClient;
49	schemaRetriever ?: SchemaRetriever;
50	database : SimulationDatabase\|undefined;
51	tokenized : boolean;
52	parser : ParserClient.ParserClient;
53	debug : boolean;
54	}
55
56	class DialogueToDSTStream extends Stream.Transform {
57	private _slotExtractor : SlotExtractor;
58	private _tokenized : boolean;
59	private _tokenizer : I18n.BaseTokenizer;
60	private _parser : ParserClient.ParserClient;
61	private _options : DialogueToDSTStreamOptions;
62
63	constructor(options : DialogueToDSTStreamOptions) {
64	super({ objectMode: true });	×
65
NEW 66	if (!options.schemaRetriever)	×
NEW 67	options.schemaRetriever = new SchemaRetriever(options.thingpediaClient, null, true);	×
NEW 68	this._slotExtractor = new SlotExtractor(options.locale, options.thingpediaClient, options.schemaRetriever, options.database);	×
69	this._tokenized = options.tokenized;	×
70	this._tokenizer = I18n.get(options.locale).getTokenizer();	×
71	this._parser = options.parser;	×
72	this._options = options;	×
73	}
74
75	private _preprocess(sentence : string, contextEntities : EntityMap) {
76	let tokenized;
77	if (this._tokenized) {	×
78	const tokens = sentence.split(' ');	×
79	const entities = Utils.makeDummyEntities(sentence);	×
80	tokenized = { tokens, entities };	×
81	} else {
82	tokenized = this._tokenizer.tokenize(sentence);	×
83	}
84	Utils.renumberEntities(tokenized, contextEntities);	×
85	return tokenized;	×
86	}
87
88	private async _checkTurn(id : string, turn : DialogueTurn, turnIndex : number) : Promise<[Record<string, string>, Record<string, string>]> {
89	let context, contextCode, contextEntities;
90	if (turnIndex > 0) {	×
91	if (turn.intermediate_context) {	×
92	context = await ThingTalkUtils.parse(turn.intermediate_context, this._options);	×
93	assert(context instanceof Ast.DialogueState);	×
94	} else {
95	context = await ThingTalkUtils.parse(turn.context!, this._options);	×
96	assert(context instanceof Ast.DialogueState);	×
97	// apply the agent prediction to the context to get the state of the dialogue before
98	// the user speaks
99	const agentPrediction = await ThingTalkUtils.parse(turn.agent_target!, this._options);	×
100	assert(agentPrediction instanceof Ast.DialogueState);	×
101	context = ThingTalkUtils.computeNewState(context, agentPrediction, 'agent');	×
102	}
103
104	const userContext = ThingTalkUtils.prepareContextForPrediction(context, 'user');	×
105	[contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(userContext);	×
106	} else {
107	context = null;	×
108	contextCode = ['null'];	×
109	contextEntities = {};	×
110	}
111
112	const { tokens, entities } = this._preprocess(turn.user, contextEntities);	×
113	const goldUserTarget = await ThingTalkUtils.parse(turn.user_target, this._options);	×
114	assert(goldUserTarget instanceof Ast.DialogueState);	×
115	const goldUserState = ThingTalkUtils.computeNewState(context, goldUserTarget, 'user');	×
116	const goldSlots = await this._slotExtractor.extractSlots(goldUserState);	×
117
118	const parsed = await this._parser.sendUtterance(tokens.join(' '), contextCode, contextEntities, {	×
119	tokenized: true,
120	skip_typechecking: true
121	});
122
123	const predictions = parsed.candidates	×
124	.filter((beam) => beam.score !== 'Infinity') // ignore exact matches	×
125	.map((beam) => beam.code);	×
126
127	if (predictions.length === 0)	×
128	return [goldSlots, {}];	×
129
130	const choice = predictions[0];	×
131
132	let predictedUserTarget;
133	try {	×
134	predictedUserTarget = await ThingTalkUtils.parsePrediction(choice, entities, this._options);	×
135	} catch(e) {
136	return [goldSlots, {}];	×
137	}
138	if (predictedUserTarget === null)	×
139	return [goldSlots, {}];	×
140
141	assert(predictedUserTarget instanceof Ast.DialogueState);	×
142	const predictedUserState = ThingTalkUtils.computeNewState(context, predictedUserTarget, 'user');	×
143	let predictedSlots;
144	try {	×
145	predictedSlots = await this._slotExtractor.extractSlots(predictedUserState);	×
146	} catch(e) {
147	console.error(predictedUserTarget.prettyprint());	×
148	throw e;	×
149	}
150
151	return [goldSlots, predictedSlots];	×
152	}
153
154	private async _doDialogue(dlg : ParsedDialogue) : Promise<[string, Record<number, { turn_belief : Record<string, string>, pred_bs_ptr : Record<string, string> }>]> {
155	const output : Record<number, { turn_belief : Record<string, string>, pred_bs_ptr : Record<string, string> }> = [];	×
156	for (let i = 0; i < dlg.length; i++) {	×
157	const turn = dlg[i];	×
158
159	const [goldSlots, predictedSlots] = await this._checkTurn(dlg.id, turn, i);	×
160	output[i] = {	×
161	turn_belief: goldSlots,
162	pred_bs_ptr: predictedSlots
163	};
164	}
165
166	return [dlg.id, output];	×
167	}
168
169	_transform(dlg : ParsedDialogue, encoding : BufferEncoding, callback : (error : Error\|null, result ?: unknown) => void) {
170	this._doDialogue(dlg).then((result) => callback(null, result), callback);	×
171	}
172
173	_flush(callback : () => void) {
174	process.nextTick(callback);	×
175	}
176	}
177
178	export function initArgparse(subparsers : argparse.SubParser) {	90✔
179	const parser = subparsers.add_parser('extract-predicted-slots', {	90✔
180	add_help: true,
181	description: "Transform a dialog input file in ThingTalk format into a dialogue state tracking prediction file."
182	});
183	parser.add_argument('-o', '--output', {	90✔
184	required: false,
185	default: process.stdout,
186	type: fs.createWriteStream,
187	help: "Write results to this file instead of stdout"
188	});
189	parser.add_argument('-l', '--locale', {	90✔
190	required: false,
191	default: 'en-US',
192	help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
193	});
194	parser.add_argument('--url', {	90✔
195	required: false,
196	help: "URL of the server to evaluate. Use a file:// URL pointing to a model directory to evaluate using a local instance of genienlp",
197	default: 'http://127.0.0.1:8400',
198	});
199	parser.add_argument('--tokenized', {	90✔
200	required: false,
201	action: 'store_true',
202	default: false,
203	help: "The dataset is already tokenized."
204	});
205	parser.add_argument('--no-tokenized', {	90✔
206	required: false,
207	dest: 'tokenized',
208	action: 'store_false',
209	help: "The dataset is not already tokenized (this is the default)."
210	});
211	parser.add_argument('--thingpedia', {	90✔
212	required: true,
213	help: 'Path to ThingTalk file containing class definitions.'
214	});
215	parser.add_argument('input_file', {	90✔
216	nargs: '+',
217	type: maybeCreateReadStream,
218	help: 'Input datasets to evaluate (in dialog format); use - for standard input'
219	});
220	parser.add_argument('--debug', {	90✔
221	action: 'store_true',
222	help: 'Enable debugging.',
223	default: true
224	});
225	parser.add_argument('--no-debug', {	90✔
226	action: 'store_false',
227	dest: 'debug',
228	help: 'Disable debugging.',
229	});
230	parser.add_argument('--database-file', {	90✔
231	required: false,
232	help: `Path to a file pointing to JSON databases used to simulate queries.`,
233	});
234	}
235
236	export async function execute(args : any) {	90✔
237	const tpClient = new Tp.FileClient(args);	×
238	const parser = ParserClient.get(args.url, args.locale);	×
239	await parser.start();	×
240
241	let database;
242	if (args.database_file) {	×
243	database = new MultiJSONDatabase(args.database_file);	×
244	await database.load();	×
245	}
246
247	readAllLines(args.input_file, '====')	×
248	.pipe(new DialogueParser())
249	.pipe(new DialogueToDSTStream({
250	locale: args.locale,
251	debug: args.debug,
252	tokenized: args.tokenized,
253	thingpediaClient: tpClient,
254	database: database,
255	parser: parser,
256	}))
257	.pipe((JSONStream as any /* FIXME bad type declaration */).stringifyObject(undefined, undefined, undefined, 2))
258	.pipe(args.output);
259
260	await StreamUtils.waitFinish(args.output);	×
261
262	await parser.stop();	×
263	}

stanford-oval / genie-toolkit / 1987

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous