stanford-oval / genie-toolkit / 727

pending completion

Build # 727

Build Type

Pull #156

travis-ci-com

Committed by

web-flow

Commit Message

Merge beef23f19 into 4ae9652e1

Pull Request Pull Request #156: webqa-make-string-dataset: chunk long strings before passing them to …

Run Details

1275 of 4713 branches covered (27.05%)

Branch coverage included in aggregate %.

0 of 12 new or added lines in 1 file covered. (0.0%)

51 existing lines in 1 file now uncovered.

3972 of 9530 relevant lines covered (41.68%)

16084.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

10.81

/tool/webqa-make-string-datasets.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const fs = require('fs');
const util = require('util');
const path = require('path');
const ThingTalk = require('thingtalk');
const csvstringify = require('csv-stringify');

const Tokenizer = require('../lib/tokenizer');
const StreamUtils = require('../lib/stream-utils');
const { makeMetadata } = require('./lib/webqa-metadata');

class ParamDatasetGenerator {
    constructor(locale, debug, className) {
        this._locale = locale;
        this._debug = debug;
        this._className = className;
        this._prefix = `${className}:`;

        this._meta = {};
        this._stringFiles = new Map;

        this._tokenizer = Tokenizer.get('local', true);
    }

    _getStringFile(stringFileName, isEntity) {
        if (this._stringFiles.has(stringFileName))
            return this._stringFiles.get(stringFileName).file;
        const file = isEntity ? [] : new Map;
        this._stringFiles.set(stringFileName, { file, isEntity });
        return file;
    }

    _addString(stringFileName, value) {
        const stringFile = this._getStringFile(stringFileName, false);
        stringFile.set(value, (stringFile.get(value) || 0) + 1);
    }

    async init(thingpedia) {
        const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));
        assert(library.isLibrary && library.classes.length === 1);
        const classDef = library.classes[0];

        for (let fn in classDef.queries) {
            const fndef = classDef.queries[fn];
            const args = [];
            for (let arg of fndef.iterateArguments())
                args.push(arg);
            this._meta[fn] = {
                extends: fndef.extends,
                fields: makeMetadata(this._className, args)
            };
        }
    }

    _processObject(value, type, visitedTypes = new Set) {
        const typemeta = this._meta[type];
        if (!typemeta)
            return;

        // if the same base class comes through multiple paths, avoid
        // visiting twice and duplicating the objects
        if (visitedTypes.has(type))
            return;
        visitedTypes.add(type);

        for (let base of typemeta.extends)
            this._processObject(value, base, visitedTypes);

        // if this entity has a name, add it to the string file
        if (type !== 'Thing' && type !== 'Intangible' && value.name)
            this._getStringFile(type, true).push({ name: value.name, value: value['@id'] });

        for (let field in typemeta.fields) {
            const expectedType = typemeta.fields[field];
            this._processField(value[field], [type, field], expectedType);
        }
    }

    _processField(value, path, expectedType) {
        if (!value)
            return;
        if (expectedType.isArray) {
            assert(Array.isArray(value));
            const innerExpected = { isArray: false, type: expectedType.type };
            for (let elem of value)
                this._processField(elem, path, innerExpected);
            return;
        }

        if (typeof expectedType.type === 'string') {
            // entity of builtin type

            if (expectedType.type === 'tt:String') {
                assert(typeof value === 'string');
                this._addString(path.join('_'), value);
            }
        } else {
            // compound type

            for (let field in expectedType.type) {
                path.push(field);
                this._processField(value[field], path, expectedType.type[field]);
                path.pop();
            }
        }
    }

    run(data) {
        for (let type in data) {
            for (let objId in data[type])
                this._processObject(data[type][objId], data[type][objId]['@type']);
        }
    }

    async _tokenizeOne(str) {
        // chunk long sentences into chunks of at most 1000 characters

        if (str.length < 500)
            return this._tokenizer.tokenize(this._locale, str);

        // very crude chunking, might break a word in the middle
        // this is mostly ok anyway cause we'll pick a substring anyway
        const output = [];
        for (let i = 0; i < str.length; i += 500) {
            const chunk = str.substring(i, i+500);
            const { tokens } = await this._tokenizer.tokenize(this._locale, chunk);
            // refuse to do anything if we have any entity (we'll drop this string later anyway)
            if (tokens.some((tok) => /^[A-Z]/.test(tok)))
                return { tokens: [], entities: {} };

            output.push(...tokens);
        }

        return { tokens: output, entities: {} };
    }

    async _tokenizeAll(strings) {
        let output = [];
        for (let i = 0; i < strings.length; i += 100) {
            console.log(`${i}/${strings.length}`);
            const slice = strings.slice(i, i+100);
            const tokenized = await Promise.all(slice.map((str) => this._tokenizeOne(str)));
            output.push(...tokenized);
        }
        return output;
    }

    async output(outputDir, manifestFile, appendManifest) {
        await util.promisify(fs.mkdir)(outputDir, { recursive: true });

        const manifestDir = path.dirname(manifestFile);
        const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });

        for (let [fileId, fileContent] of this._stringFiles) {
            const outputpath = path.resolve(outputDir, this._prefix + fileId + (fileContent.isEntity ? '.json' : '.tsv'));

            if (fileContent.isEntity) {
                if (this._debug)
                    console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);

                const tokenized = await this._tokenizeAll(fileContent.file.map((entity) => entity.name));
                const data = [];
                for (let i = 0; i < fileContent.file.length; i++) {
                    const entity = fileContent.file[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    entity.canonical = tokens.join(' ');
                    data.push(entity);
                }

                await util.promisify(fs.writeFile)(outputpath, JSON.stringify({
                    result: 'ok', data
                }, undefined, 2), { encoding: 'utf8' });

                manifest.write(`entity\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);
            } else {
                if (this._debug)
                    console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);

                const outputfile = fs.createWriteStream(outputpath);
                const output = csvstringify({ header: false, delimiter: '\t' });
                output.pipe(outputfile);

                const strings = Array.from(fileContent.file.keys());
                const weights = Array.from(fileContent.file.values());
                const tokenized = await this._tokenizeAll(strings);

                for (let i = 0; i < strings.length; i++) {
                    const value = strings[i];
                    const weight = weights[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    output.write([value, tokens.join(' '), weight]);
                }

                output.end();
                manifest.write(`string\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);

                //await StreamUtils.waitFinish(output);
                console.log(`completed ${fileId}`);
            }
        }

        manifest.end();
        await StreamUtils.waitFinish(manifest);
        await this._tokenizer.end();
    }
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('webqa-make-string-datasets', {
            addHelp: true,
            description: "Extract string datasets from a WebQA normalized data file."
        });
        parser.addArgument(['-d', '--output-dir'], {
            required: true,
        });
        parser.addArgument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.addArgument(['-l', '--locale'], {
            required: false,
            defaultValue: 'en-US',
            help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
        });
        parser.addArgument('--manifest', {
            required: true,
            help: `Write a parameter dataset manifest to this location`
        });
        parser.addArgument('--append-manifest', {
            required: false,
            action: 'storeTrue',
            help: `append to the manifest instead of replacing`
        });
        parser.addArgument('--data', {
            required: true,
            help: 'Path to JSON file with normalized WebQA data.'
        });

        parser.addArgument('--debug', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Enable debugging.',
            defaultValue: true
        });
        parser.addArgument('--no-debug', {
            nargs: 0,
            action: 'storeFalse',
            dest: 'debug',
            help: 'Disable debugging.',
        });
        parser.addArgument('--class-name', {
            required: false,
            help: 'The name of the device class, used for decide class-specific types'
        });
    },

    async execute(args) {
        const generator = new ParamDatasetGenerator(args.locale, args.debug, args.class_name);
        await generator.init(args.thingpedia);

        const data = JSON.parse(await util.promisify(fs.readFile)(args.data, { encoding: 'utf8' }));
        generator.run(data);

        await generator.output(args.output_dir, args.manifest, args.append_manifest);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	46✔
13	const fs = require('fs');	46✔
14	const util = require('util');	46✔
15	const path = require('path');	46✔
16	const ThingTalk = require('thingtalk');	46✔
17	const csvstringify = require('csv-stringify');	46✔
18
19	const Tokenizer = require('../lib/tokenizer');	46✔
20	const StreamUtils = require('../lib/stream-utils');	46✔
21	const { makeMetadata } = require('./lib/webqa-metadata');	46✔
22
23	class ParamDatasetGenerator {
24	constructor(locale, debug, className) {
UNCOV 25	this._locale = locale;	×
UNCOV 26	this._debug = debug;	×
27	this._className = className;	×
28	this._prefix = `${className}:`;	×
29
30	this._meta = {};	×
UNCOV 31	this._stringFiles = new Map;	×
32
33	this._tokenizer = Tokenizer.get('local', true);	×
34	}
35
36	_getStringFile(stringFileName, isEntity) {
UNCOV 37	if (this._stringFiles.has(stringFileName))	×
UNCOV 38	return this._stringFiles.get(stringFileName).file;	×
39	const file = isEntity ? [] : new Map;	×
40	this._stringFiles.set(stringFileName, { file, isEntity });	×
41	return file;	×
42	}
43
44	_addString(stringFileName, value) {
UNCOV 45	const stringFile = this._getStringFile(stringFileName, false);	×
UNCOV 46	stringFile.set(value, (stringFile.get(value) \|\| 0) + 1);	×
47	}
48
49	async init(thingpedia) {
UNCOV 50	const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));	×
UNCOV 51	assert(library.isLibrary && library.classes.length === 1);	×
52	const classDef = library.classes[0];	×
53
54	for (let fn in classDef.queries) {	×
UNCOV 55	const fndef = classDef.queries[fn];	×
56	const args = [];	×
57	for (let arg of fndef.iterateArguments())	×
58	args.push(arg);	×
59	this._meta[fn] = {	×
60	extends: fndef.extends,
61	fields: makeMetadata(this._className, args)
62	};
63	}
64	}
65
66	_processObject(value, type, visitedTypes = new Set) {	×
UNCOV 67	const typemeta = this._meta[type];	×
UNCOV 68	if (!typemeta)	×
69	return;	×
70
71	// if the same base class comes through multiple paths, avoid
72	// visiting twice and duplicating the objects
UNCOV 73	if (visitedTypes.has(type))	×
UNCOV 74	return;	×
75	visitedTypes.add(type);	×
76
77	for (let base of typemeta.extends)	×
UNCOV 78	this._processObject(value, base, visitedTypes);	×
79
80	// if this entity has a name, add it to the string file
UNCOV 81	if (type !== 'Thing' && type !== 'Intangible' && value.name)	×
UNCOV 82	this._getStringFile(type, true).push({ name: value.name, value: value['@id'] });	×
83
84	for (let field in typemeta.fields) {	×
UNCOV 85	const expectedType = typemeta.fields[field];	×
86	this._processField(value[field], [type, field], expectedType);	×
87	}
88	}
89
90	_processField(value, path, expectedType) {
UNCOV 91	if (!value)	×
UNCOV 92	return;	×
93	if (expectedType.isArray) {	×
94	assert(Array.isArray(value));	×
95	const innerExpected = { isArray: false, type: expectedType.type };	×
96	for (let elem of value)	×
97	this._processField(elem, path, innerExpected);	×
98	return;	×
99	}
100
UNCOV 101	if (typeof expectedType.type === 'string') {	×
102	// entity of builtin type
103
UNCOV 104	if (expectedType.type === 'tt:String') {	×
UNCOV 105	assert(typeof value === 'string');	×
106	this._addString(path.join('_'), value);	×
107	}
108	} else {
109	// compound type
110
UNCOV 111	for (let field in expectedType.type) {	×
UNCOV 112	path.push(field);	×
113	this._processField(value[field], path, expectedType.type[field]);	×
114	path.pop();	×
115	}
116	}
117	}
118
119	run(data) {
UNCOV 120	for (let type in data) {	×
UNCOV 121	for (let objId in data[type])	×
122	this._processObject(data[type][objId], data[type][objId]['@type']);	×
123	}
124	}
125
126	async _tokenizeOne(str) {
127	// chunk long sentences into chunks of at most 1000 characters
128
NEW 129	if (str.length < 500)	×
NEW 130	return this._tokenizer.tokenize(this._locale, str);	×
131
132	// very crude chunking, might break a word in the middle
133	// this is mostly ok anyway cause we'll pick a substring anyway
NEW 134	const output = [];	×
NEW 135	for (let i = 0; i < str.length; i += 500) {	×
NEW 136	const chunk = str.substring(i, i+500);	×
NEW 137	const { tokens } = await this._tokenizer.tokenize(this._locale, chunk);	×
138	// refuse to do anything if we have any entity (we'll drop this string later anyway)
NEW 139	if (tokens.some((tok) => /^[A-Z]/.test(tok)))	×
NEW 140	return { tokens: [], entities: {} };	×
141
NEW 142	output.push(...tokens);	×
143	}
144
NEW 145	return { tokens: output, entities: {} };	×
146	}
147
148	async _tokenizeAll(strings) {
149	let output = [];	×
150	for (let i = 0; i < strings.length; i += 100) {	×
151	console.log(`${i}/${strings.length}`);	×
152	const slice = strings.slice(i, i+100);	×
NEW 153	const tokenized = await Promise.all(slice.map((str) => this._tokenizeOne(str)));	×
154	output.push(...tokenized);	×
155	}
156	return output;	×
157	}
158
159	async output(outputDir, manifestFile, appendManifest) {
UNCOV 160	await util.promisify(fs.mkdir)(outputDir, { recursive: true });	×
161
162	const manifestDir = path.dirname(manifestFile);	×
UNCOV 163	const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });	×
164
165	for (let [fileId, fileContent] of this._stringFiles) {	×
UNCOV 166	const outputpath = path.resolve(outputDir, this._prefix + fileId + (fileContent.isEntity ? '.json' : '.tsv'));	×
167
168	if (fileContent.isEntity) {	×
UNCOV 169	if (this._debug)	×
170	console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);	×
171
172	const tokenized = await this._tokenizeAll(fileContent.file.map((entity) => entity.name));	×
UNCOV 173	const data = [];	×
174	for (let i = 0; i < fileContent.file.length; i++) {	×
175	const entity = fileContent.file[i];	×
176	const tokens = tokenized[i].tokens;	×
177
178	// if some tokens are uppercase, they are entities, like NUMBER_0,
179	// in which case we ignore this value
UNCOV 180	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
UNCOV 181	continue;	×
182
183	entity.canonical = tokens.join(' ');	×
UNCOV 184	data.push(entity);	×
185	}
186
UNCOV 187	await util.promisify(fs.writeFile)(outputpath, JSON.stringify({	×
188	result: 'ok', data
189	}, undefined, 2), { encoding: 'utf8' });
190
UNCOV 191	manifest.write(`entity\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
192	} else {
193	if (this._debug)	×
UNCOV 194	console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);	×
195
196	const outputfile = fs.createWriteStream(outputpath);	×
UNCOV 197	const output = csvstringify({ header: false, delimiter: '\t' });	×
198	output.pipe(outputfile);	×
199
200	const strings = Array.from(fileContent.file.keys());	×
UNCOV 201	const weights = Array.from(fileContent.file.values());	×
202	const tokenized = await this._tokenizeAll(strings);	×
203
204	for (let i = 0; i < strings.length; i++) {	×
UNCOV 205	const value = strings[i];	×
206	const weight = weights[i];	×
207	const tokens = tokenized[i].tokens;	×
208
209	// if some tokens are uppercase, they are entities, like NUMBER_0,
210	// in which case we ignore this value
NEW UNCOV 211	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
UNCOV 212	continue;	×
213
214	output.write([value, tokens.join(' '), weight]);	×
215	}
216
UNCOV 217	output.end();	×
UNCOV 218	manifest.write(`string\t${this._prefix}${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
219
220	//await StreamUtils.waitFinish(output);
UNCOV 221	console.log(`completed ${fileId}`);	×
222	}
223	}
224
UNCOV 225	manifest.end();	×
UNCOV 226	await StreamUtils.waitFinish(manifest);	×
227	await this._tokenizer.end();	×
228	}
229	}
230
231	module.exports = {	46✔
232	initArgparse(subparsers) {
233	const parser = subparsers.addParser('webqa-make-string-datasets', {	46✔
234	addHelp: true,
235	description: "Extract string datasets from a WebQA normalized data file."
236	});
237	parser.addArgument(['-d', '--output-dir'], {	46✔
238	required: true,
239	});
240	parser.addArgument('--thingpedia', {	46✔
241	required: true,
242	help: 'Path to ThingTalk file containing class definitions.'
243	});
244	parser.addArgument(['-l', '--locale'], {	46✔
245	required: false,
246	defaultValue: 'en-US',
247	help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
248	});
249	parser.addArgument('--manifest', {	46✔
250	required: true,
251	help: `Write a parameter dataset manifest to this location`
252	});
253	parser.addArgument('--append-manifest', {	46✔
254	required: false,
255	action: 'storeTrue',
256	help: `append to the manifest instead of replacing`
257	});
258	parser.addArgument('--data', {	46✔
259	required: true,
260	help: 'Path to JSON file with normalized WebQA data.'
261	});
262
263	parser.addArgument('--debug', {	46✔
264	nargs: 0,
265	action: 'storeTrue',
266	help: 'Enable debugging.',
267	defaultValue: true
268	});
269	parser.addArgument('--no-debug', {	46✔
270	nargs: 0,
271	action: 'storeFalse',
272	dest: 'debug',
273	help: 'Disable debugging.',
274	});
275	parser.addArgument('--class-name', {	46✔
276	required: false,
277	help: 'The name of the device class, used for decide class-specific types'
278	});
279	},
280
281	async execute(args) {
UNCOV 282	const generator = new ParamDatasetGenerator(args.locale, args.debug, args.class_name);	×
UNCOV 283	await generator.init(args.thingpedia);	×
284
285	const data = JSON.parse(await util.promisify(fs.readFile)(args.data, { encoding: 'utf8' }));	×
UNCOV 286	generator.run(data);	×
287
288	await generator.output(args.output_dir, args.manifest, args.append_manifest);	×
289	}
290	};

stanford-oval / genie-toolkit / 727

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous