stanford-oval / genie-toolkit / 734

pending completion

Build # 734

Build Type

Pull #152

travis-ci-com

Committed by

web-flow

Commit Message

Merge 6ee8891f0 into e8599c64d

Pull Request Pull Request #152: Wip/multilanguage

Run Details

1349 of 4759 branches covered (28.35%)

Branch coverage included in aggregate %.

81 of 102 new or added lines in 6 files covered. (79.41%)

64 existing lines in 2 files now uncovered.

4075 of 9606 relevant lines covered (42.42%)

17363.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

11.29

/tool/webqa-make-string-datasets.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const fs = require('fs');
const util = require('util');
const path = require('path');
const ThingTalk = require('thingtalk');
const csvstringify = require('csv-stringify');

const Tokenizer = require('../lib/tokenizer');
const StreamUtils = require('../lib/stream-utils');
const { makeMetadata } = require('./lib/webqa-metadata');

const MAX_CHARS = 200;

class ParamDatasetGenerator {
    constructor(locale, debug, className) {
        this._locale = locale;
        this._debug = debug;
        this._className = className;
        this._prefix = `${className}:`;

        this._meta = {};
        this._stringFiles = new Map;

        this._tokenizer = Tokenizer.get('local', true);
    }

    _getStringFile(stringFileName, isEntity) {
        if (this._stringFiles.has(stringFileName))
            return this._stringFiles.get(stringFileName).file;
        const file = isEntity ? [] : new Map;
        this._stringFiles.set(stringFileName, { file, isEntity });
        return file;
    }

    _addString(stringFileName, value) {
        const stringFile = this._getStringFile(stringFileName, false);
        stringFile.set(value, (stringFile.get(value) || 0) + 1);
    }

    async init(thingpedia) {
        const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));
        assert(library.isLibrary && library.classes.length === 1);
        const classDef = library.classes[0];

        for (let fn in classDef.queries) {
            const fndef = classDef.queries[fn];
            const args = [];
            for (let arg of fndef.iterateArguments())
                args.push(arg);
            this._meta[fn] = {
                extends: fndef.extends,
                fields: makeMetadata(this._className, args)
            };
        }
    }

    _processObject(value, type, visitedTypes = new Set) {
        const typemeta = this._meta[type];
        if (!typemeta)
            return;

        // if the same base class comes through multiple paths, avoid
        // visiting twice and duplicating the objects
        if (visitedTypes.has(type))
            return;
        visitedTypes.add(type);

        for (let base of typemeta.extends)
            this._processObject(value, base, visitedTypes);

        // if this entity has a name, add it to the string file
        if (type !== 'Thing' && type !== 'Intangible' && value.name)
            this._getStringFile(type, true).push({ name: value.name, value: value['@id'] });

        for (let field in typemeta.fields) {
            const expectedType = typemeta.fields[field];
            this._processField(value[field], [type, field], expectedType);
        }
    }

    _processField(value, path, expectedType) {
        if (!value)
            return;
        if (expectedType.isArray) {
            assert(Array.isArray(value));
            const innerExpected = { isArray: false, type: expectedType.type };
            for (let elem of value)
                this._processField(elem, path, innerExpected);
            return;
        }

        if (typeof expectedType.type === 'string') {
            // entity of builtin type

            if (expectedType.type === 'tt:String') {
                assert(typeof value === 'string');
                this._addString(path.join('_'), value);
            }
        } else {
            // compound type

            for (let field in expectedType.type) {
                path.push(field);
                this._processField(value[field], path, expectedType.type[field]);
                path.pop();
            }
        }
    }

    run(data) {
        for (let type in data) {
            for (let objId in data[type])
                this._processObject(data[type][objId], data[type][objId]['@type']);
        }
    }

    async _tokenizeOne(str) {
        // chunk long sentences into chunks of at most 1000 characters

        if (str.length < MAX_CHARS)
            return this._tokenizer.tokenize(this._locale, str);

        // very crude chunking, might break a word in the middle
        // this is mostly ok anyway cause we'll pick a substring anyway
        const output = [];
        for (let i = 0; i < str.length; i += MAX_CHARS) {
            const chunk = str.substring(i, i+MAX_CHARS);
            const { tokens } = await this._tokenizer.tokenize(this._locale, chunk);
            // refuse to do anything if we have any entity (we'll drop this string later anyway)
            if (tokens.some((tok) => /^[A-Z]/.test(tok)))
                return { tokens: [], entities: {} };

            output.push(...tokens);
        }

        return { tokens: output, entities: {} };
    }

    async _tokenizeAll(strings) {
        let output = [];
        for (let i = 0; i < strings.length; i += 100) {
            console.log(`${i}/${strings.length}`);
            const slice = strings.slice(i, i+100);
            const tokenized = await Promise.all(slice.map((str) => this._tokenizeOne(str)));
            output.push(...tokenized);
        }
        return output;
    }

    async output(outputDir, manifestFile, appendManifest) {
        await util.promisify(fs.mkdir)(outputDir, { recursive: true });

        const manifestDir = path.dirname(manifestFile);
        const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });

        for (let [fileId, fileContent] of this._stringFiles) {
            const outputpath = path.resolve(outputDir, this._prefix + fileId + (fileContent.isEntity ? '.json' : '.tsv'));

            if (fileContent.isEntity) {
                if (this._debug)
                    console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);

                const tokenized = await this._tokenizeAll(fileContent.file.map((entity) => entity.name));
                const data = [];
                for (let i = 0; i < fileContent.file.length; i++) {
                    const entity = fileContent.file[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    entity.canonical = tokens.join(' ');
                    data.push(entity);
                }

                await util.promisify(fs.writeFile)(outputpath, JSON.stringify({
                    result: 'ok', data
                }, undefined, 2), { encoding: 'utf8' });

                manifest.write(`entity\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);
            } else {
                if (this._debug)
                    console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);

                const outputfile = fs.createWriteStream(outputpath);
                const output = csvstringify({ header: false, delimiter: '\t' });
                output.pipe(outputfile);

                const strings = Array.from(fileContent.file.keys());
                const weights = Array.from(fileContent.file.values());
                const tokenized = await this._tokenizeAll(strings);

                for (let i = 0; i < strings.length; i++) {
                    const value = strings[i];
                    const weight = weights[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    output.write([value, tokens.join(' '), weight]);
                }

                output.end();
                manifest.write(`string\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);

                //await StreamUtils.waitFinish(output);
                console.log(`completed ${fileId}`);
            }
        }

        manifest.end();
        await StreamUtils.waitFinish(manifest);
        await this._tokenizer.end();
    }
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('webqa-make-string-datasets', {
            addHelp: true,
            description: "Extract string datasets from a WebQA normalized data file."
        });
        parser.addArgument(['-d', '--output-dir'], {
            required: true,
        });
        parser.addArgument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.addArgument(['-l', '--locale'], {
            required: false,
            defaultValue: 'en-US',
            help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
        });
        parser.addArgument('--manifest', {
            required: true,
            help: `Write a parameter dataset manifest to this location`
        });
        parser.addArgument('--append-manifest', {
            required: false,
            action: 'storeTrue',
            help: `append to the manifest instead of replacing`
        });
        parser.addArgument('--data', {
            required: true,
            help: 'Path to JSON file with normalized WebQA data.'
        });

        parser.addArgument('--debug', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Enable debugging.',
            defaultValue: true
        });
        parser.addArgument('--no-debug', {
            nargs: 0,
            action: 'storeFalse',
            dest: 'debug',
            help: 'Disable debugging.',
        });
        parser.addArgument('--class-name', {
            required: false,
            help: 'The name of the device class, used for decide class-specific types'
        });
    },

    async execute(args) {
        const generator = new ParamDatasetGenerator(args.locale, args.debug, args.class_name);
        await generator.init(args.thingpedia);

        const data = JSON.parse(await util.promisify(fs.readFile)(args.data, { encoding: 'utf8' }));
        generator.run(data);

        await generator.output(args.output_dir, args.manifest, args.append_manifest);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	50✔
13	const fs = require('fs');	50✔
14	const util = require('util');	50✔
15	const path = require('path');	50✔
16	const ThingTalk = require('thingtalk');	50✔
17	const csvstringify = require('csv-stringify');	50✔
18
19	const Tokenizer = require('../lib/tokenizer');	50✔
20	const StreamUtils = require('../lib/stream-utils');	50✔
21	const { makeMetadata } = require('./lib/webqa-metadata');	50✔
22
23	const MAX_CHARS = 200;	50✔
24
25	class ParamDatasetGenerator {
26	constructor(locale, debug, className) {
27	this._locale = locale;	×
28	this._debug = debug;	×
UNCOV 29	this._className = className;	×
30	this._prefix = `${className}:`;	×
31
UNCOV 32	this._meta = {};	×
33	this._stringFiles = new Map;	×
34
UNCOV 35	this._tokenizer = Tokenizer.get('local', true);	×
36	}
37
38	_getStringFile(stringFileName, isEntity) {
39	if (this._stringFiles.has(stringFileName))	×
40	return this._stringFiles.get(stringFileName).file;	×
41	const file = isEntity ? [] : new Map;	×
UNCOV 42	this._stringFiles.set(stringFileName, { file, isEntity });	×
UNCOV 43	return file;	×
44	}
45
46	_addString(stringFileName, value) {
UNCOV 47	const stringFile = this._getStringFile(stringFileName, false);	×
UNCOV 48	stringFile.set(value, (stringFile.get(value) \|\| 0) + 1);	×
49	}
50
51	async init(thingpedia) {
52	const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));	×
UNCOV 53	assert(library.isLibrary && library.classes.length === 1);	×
54	const classDef = library.classes[0];	×
55
56	for (let fn in classDef.queries) {	×
57	const fndef = classDef.queries[fn];	×
58	const args = [];	×
59	for (let arg of fndef.iterateArguments())	×
UNCOV 60	args.push(arg);	×
UNCOV 61	this._meta[fn] = {	×
62	extends: fndef.extends,
63	fields: makeMetadata(this._className, args)
64	};
65	}
66	}
67
68	_processObject(value, type, visitedTypes = new Set) {	×
69	const typemeta = this._meta[type];	×
UNCOV 70	if (!typemeta)	×
UNCOV 71	return;	×
72
73	// if the same base class comes through multiple paths, avoid
74	// visiting twice and duplicating the objects
75	if (visitedTypes.has(type))	×
UNCOV 76	return;	×
77	visitedTypes.add(type);	×
78
UNCOV 79	for (let base of typemeta.extends)	×
UNCOV 80	this._processObject(value, base, visitedTypes);	×
81
82	// if this entity has a name, add it to the string file
UNCOV 83	if (type !== 'Thing' && type !== 'Intangible' && value.name)	×
84	this._getStringFile(type, true).push({ name: value.name, value: value['@id'] });	×
85
86	for (let field in typemeta.fields) {	×
UNCOV 87	const expectedType = typemeta.fields[field];	×
UNCOV 88	this._processField(value[field], [type, field], expectedType);	×
89	}
90	}
91
92	_processField(value, path, expectedType) {
93	if (!value)	×
94	return;	×
95	if (expectedType.isArray) {	×
96	assert(Array.isArray(value));	×
97	const innerExpected = { isArray: false, type: expectedType.type };	×
98	for (let elem of value)	×
UNCOV 99	this._processField(elem, path, innerExpected);	×
UNCOV 100	return;	×
101	}
102
UNCOV 103	if (typeof expectedType.type === 'string') {	×
104	// entity of builtin type
105
106	if (expectedType.type === 'tt:String') {	×
UNCOV 107	assert(typeof value === 'string');	×
UNCOV 108	this._addString(path.join('_'), value);	×
109	}
110	} else {
111	// compound type
112
113	for (let field in expectedType.type) {	×
114	path.push(field);	×
UNCOV 115	this._processField(value[field], path, expectedType.type[field]);	×
UNCOV 116	path.pop();	×
117	}
118	}
119	}
120
121	run(data) {
122	for (let type in data) {	×
UNCOV 123	for (let objId in data[type])	×
UNCOV 124	this._processObject(data[type][objId], data[type][objId]['@type']);	×
125	}
126	}
127
128	async _tokenizeOne(str) {
129	// chunk long sentences into chunks of at most 1000 characters
130
131	if (str.length < MAX_CHARS)	×
132	return this._tokenizer.tokenize(this._locale, str);	×
133
134	// very crude chunking, might break a word in the middle
135	// this is mostly ok anyway cause we'll pick a substring anyway
UNCOV 136	const output = [];	×
UNCOV 137	for (let i = 0; i < str.length; i += MAX_CHARS) {	×
138	const chunk = str.substring(i, i+MAX_CHARS);	×
UNCOV 139	const { tokens } = await this._tokenizer.tokenize(this._locale, chunk);	×
140	// refuse to do anything if we have any entity (we'll drop this string later anyway)
141	if (tokens.some((tok) => /^[A-Z]/.test(tok)))	×
UNCOV 142	return { tokens: [], entities: {} };	×
143
144	output.push(...tokens);	×
145	}
146
147	return { tokens: output, entities: {} };	×
148	}
149
150	async _tokenizeAll(strings) {
151	let output = [];	×
152	for (let i = 0; i < strings.length; i += 100) {	×
153	console.log(`${i}/${strings.length}`);	×
154	const slice = strings.slice(i, i+100);	×
UNCOV 155	const tokenized = await Promise.all(slice.map((str) => this._tokenizeOne(str)));	×
UNCOV 156	output.push(...tokenized);	×
157	}
158	return output;	×
159	}
160
161	async output(outputDir, manifestFile, appendManifest) {
162	await util.promisify(fs.mkdir)(outputDir, { recursive: true });	×
163
UNCOV 164	const manifestDir = path.dirname(manifestFile);	×
165	const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });	×
166
UNCOV 167	for (let [fileId, fileContent] of this._stringFiles) {	×
UNCOV 168	const outputpath = path.resolve(outputDir, this._prefix + fileId + (fileContent.isEntity ? '.json' : '.tsv'));	×
169
UNCOV 170	if (fileContent.isEntity) {	×
171	if (this._debug)	×
172	console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);	×
173
174	const tokenized = await this._tokenizeAll(fileContent.file.map((entity) => entity.name));	×
175	const data = [];	×
176	for (let i = 0; i < fileContent.file.length; i++) {	×
UNCOV 177	const entity = fileContent.file[i];	×
178	const tokens = tokenized[i].tokens;	×
179
180	// if some tokens are uppercase, they are entities, like NUMBER_0,
181	// in which case we ignore this value
182	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
183	continue;	×
184
185	entity.canonical = tokens.join(' ');	×
UNCOV 186	data.push(entity);	×
187	}
188
189	await util.promisify(fs.writeFile)(outputpath, JSON.stringify({	×
190	result: 'ok', data
191	}, undefined, 2), { encoding: 'utf8' });
192
UNCOV 193	manifest.write(`entity\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
194	} else {
195	if (this._debug)	×
196	console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);	×
197
UNCOV 198	const outputfile = fs.createWriteStream(outputpath);	×
199	const output = csvstringify({ header: false, delimiter: '\t' });	×
UNCOV 200	output.pipe(outputfile);	×
201
UNCOV 202	const strings = Array.from(fileContent.file.keys());	×
203	const weights = Array.from(fileContent.file.values());	×
204	const tokenized = await this._tokenizeAll(strings);	×
205
UNCOV 206	for (let i = 0; i < strings.length; i++) {	×
UNCOV 207	const value = strings[i];	×
UNCOV 208	const weight = weights[i];	×
UNCOV 209	const tokens = tokenized[i].tokens;	×
210
211	// if some tokens are uppercase, they are entities, like NUMBER_0,
212	// in which case we ignore this value
UNCOV 213	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
UNCOV 214	continue;	×
215
UNCOV 216	output.write([value, tokens.join(' '), weight]);	×
217	}
218
UNCOV 219	output.end();	×
UNCOV 220	manifest.write(`string\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
221
222	//await StreamUtils.waitFinish(output);
UNCOV 223	console.log(`completed ${fileId}`);	×
224	}
225	}
226
UNCOV 227	manifest.end();	×
UNCOV 228	await StreamUtils.waitFinish(manifest);	×
UNCOV 229	await this._tokenizer.end();	×
230	}
231	}
232
233	module.exports = {	50✔
234	initArgparse(subparsers) {
235	const parser = subparsers.addParser('webqa-make-string-datasets', {	50✔
236	addHelp: true,
237	description: "Extract string datasets from a WebQA normalized data file."
238	});
239	parser.addArgument(['-d', '--output-dir'], {	50✔
240	required: true,
241	});
242	parser.addArgument('--thingpedia', {	50✔
243	required: true,
244	help: 'Path to ThingTalk file containing class definitions.'
245	});
246	parser.addArgument(['-l', '--locale'], {	50✔
247	required: false,
248	defaultValue: 'en-US',
249	help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
250	});
251	parser.addArgument('--manifest', {	50✔
252	required: true,
253	help: `Write a parameter dataset manifest to this location`
254	});
255	parser.addArgument('--append-manifest', {	50✔
256	required: false,
257	action: 'storeTrue',
258	help: `append to the manifest instead of replacing`
259	});
260	parser.addArgument('--data', {	50✔
261	required: true,
262	help: 'Path to JSON file with normalized WebQA data.'
263	});
264
265	parser.addArgument('--debug', {	50✔
266	nargs: 0,
267	action: 'storeTrue',
268	help: 'Enable debugging.',
269	defaultValue: true
270	});
271	parser.addArgument('--no-debug', {	50✔
272	nargs: 0,
273	action: 'storeFalse',
274	dest: 'debug',
275	help: 'Disable debugging.',
276	});
277	parser.addArgument('--class-name', {	50✔
278	required: false,
279	help: 'The name of the device class, used for decide class-specific types'
280	});
281	},
282
283	async execute(args) {
UNCOV 284	const generator = new ParamDatasetGenerator(args.locale, args.debug, args.class_name);	×
UNCOV 285	await generator.init(args.thingpedia);	×
286
UNCOV 287	const data = JSON.parse(await util.promisify(fs.readFile)(args.data, { encoding: 'utf8' }));	×
UNCOV 288	generator.run(data);	×
289
UNCOV 290	await generator.output(args.output_dir, args.manifest, args.append_manifest);	×
291	}
292	};

stanford-oval / genie-toolkit / 734

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous