1434

pending completion

Build # 1434

Build Type

Pull #378

travis-ci-com

Committed by

web-flow

Commit Message

Merge c5f47ade8 into cf078a09c

Pull Request Pull Request #378: Wip/wikidata single turn

Run Details

7114 of 13591 branches covered (52.34%)

Branch coverage included in aggregate %.

98 of 627 new or added lines in 14 files covered. (15.63%)

8 existing lines in 4 files now uncovered.

15922 of 25775 relevant lines covered (61.77%)

1617306.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.05

/tool/autoqa/wikidata/make-string-datasets.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const assert = require('assert');
const fs = require('fs');
const util = require('util');
const path = require('path');
const ThingTalk = require('thingtalk');
const csvstringify = require('csv-stringify');

const I18N = require('../../../lib/i18n');
const StreamUtils = require('../../../lib/utils/stream-utils');

const {
    wikidataQuery,
    getEquivalent,
    getItemLabel
} = require('./utils');

function getElemType(type) {
    if (type.isArray)
        return getElemType(type.elem);
    return type;
}

class ParamDatasetGenerator {
    constructor(options) {
        this._locale = options.locale;
        this._debug = options.debug;
        this._maxValueLength = options.max_value_length;
        this._targetSize = options.target_size;
        this._stringFiles = new Map;
        this._tokenizer = I18N.get(options.locale).getTokenizer();
    }

    _getStringFile(stringFileName, isEntity) {
        if (this._stringFiles.has(stringFileName))
            return this._stringFiles.get(stringFileName).file;
        const file = isEntity ? [] : new Map;
        this._stringFiles.set(stringFileName, { file, isEntity });
        return file;
    }

    _addString(stringFileName, value) {
        const stringFile = this._getStringFile(stringFileName, false);
        stringFile.set(value, (stringFile.get(value) || 0) + 1);
    }

    async init(thingpedia) {
        const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));
        assert(library.isLibrary && library.classes.length === 1);
        this._classDef = library.classes[0];
    }

    async _downloadSubjectValues(fn, klass, filters, targetSize) {
        const predicate = klass === 'Q5' ? 'wdt:P31' : 'p:P31/ps:P31/wdt:P279*';
        const query = `
            SELECT DISTINCT ?subject
            WHERE {
              ?subject ${predicate} wd:${klass}; ${filters.join('; ')}.
            }
            LIMIT ${targetSize}
        `;
        const results = await wikidataQuery(query);
        for (let result of results) {
            const value = result.subject.value;
            const label = await getItemLabel(value.slice('http://www.wikidata.org/entity/'.length));
            if (!label)
                continue;
            if (/Q[0-9]+/.test(label))
                continue;
            this._getStringFile(fn, true).push({ name: label, value: value });
        }

    }

    async _downloadPropertyValues(fn, arg, klass, filters, targetSize) {
        if (arg.name === 'id')
            return ;
        const elemType = getElemType(arg.type);
        if (!elemType.isString && !(elemType.isEntity && elemType.type.startsWith('org.wikidata:')))
            return;

        const id = arg.getImplementationAnnotation('wikidata_id');

        const predicate = klass === 'Q5' ? 'wdt:P31' : 'p:P31/ps:P31/wdt:P279*';
        const query = `
            SELECT DISTINCT ?value
            WHERE {
              ?subject ${predicate} wd:${klass}; ${filters.join('; ')}.
              ?subject wdt:${id} ?value.
            }
            LIMIT ${targetSize}
        `;
        const results = await wikidataQuery(query);
        for (let result of results) {
            const value = result.value.value;
            let label;
            if (value.startsWith('http://www.wikidata.org/entity/'))
                label = await getItemLabel(value.slice('http://www.wikidata.org/entity/'.length));
            else
                label = value;
            if (!label)
                continue;
            if (/Q[0-9]+/.test(label))
                continue;
            if (elemType.isString)
                this._addString(`${fn}_${arg.name}`, label);
            else if (elemType.isEntity && elemType.type.startsWith('org.wikidata:'))
                this._getStringFile(elemType.type.slice('org.wikidata:'.length), true).push({ name: label, value: value });
        }
    }

    async _tryDownloadSubjectValues(fn, klass, triples, targetSize) {
        while (targetSize > 30) {
            try {
                await this._downloadSubjectValues(fn, klass, triples, Math.ceil(targetSize));
                return;
            } catch(e) {
                if (e.code !== 500)
                    throw e;
                targetSize /= 2;
            }
        }
    }

    async _tryDownloadPropertyValues(fn, arg, klass, triples, targetSize) {
        while (targetSize > 30) {
            try {
                await this._downloadPropertyValues(fn, arg, klass, triples, Math.ceil(targetSize));
                return;
            } catch(e) {
                if (e.code !== 500)
                    throw e;
                targetSize /= 2;
            }
        }
    }

    async run() {
        for (let fn in this._classDef.queries) {
            const fndef = this._classDef.queries[fn];
            const klass = fndef.getImplementationAnnotation('wikidata_subject');
            const equivalentClasses = klass === 'Q5' ? null : await getEquivalent(klass);
            const classes =  equivalentClasses ? [klass, ...equivalentClasses] : [klass];
            const triples = [];
            for (let arg of fndef.getImplementationAnnotation('required_properties') || [])
                triples.push(`wdt:${arg} ?${arg}`);
            for (let klass of classes) {
                await this._tryDownloadSubjectValues(fn, klass, triples, Math.ceil(this._targetSize / classes.length));
                for (let arg of fndef.iterateArguments())
                    await this._tryDownloadPropertyValues(fn, arg, klass, triples, Math.ceil(this._targetSize / classes.length));
            }
        }
    }

    _tokenizeAll(strings) {
        return strings.map((str) => this._tokenizer.tokenize(str));
    }

    async output(outputDir, manifestFile, appendManifest) {
        await util.promisify(fs.mkdir)(outputDir, { recursive: true });

        const manifestDir = path.dirname(manifestFile);
        const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });

        for (let [fileId, fileContent] of this._stringFiles) {
            const outputpath = path.resolve(outputDir, 'org.wikidata:' + fileId + (fileContent.isEntity ? '.json' : '.tsv'));

            if (fileContent.isEntity) {
                if (this._debug)
                    console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);

                const tokenized = this._tokenizeAll(fileContent.file.map((entity) => entity.name));
                const data = [];
                for (let i = 0; i < fileContent.file.length; i++) {
                    const entity = fileContent.file[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    entity.canonical = tokens.join(' ');
                    if (this._maxValueLength >= 0 && entity.canonical.length > this._maxValueLength)
                        continue;

                    data.push(entity);
                }

                await util.promisify(fs.writeFile)(outputpath, JSON.stringify({
                    result: 'ok', data
                }, undefined, 2), { encoding: 'utf8' });

                manifest.write(`entity\t${this._locale}\torg.wikidata:${fileId}\t${path.relative(manifestDir, outputpath)}\n`);
            } else {
                if (this._debug)
                    console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);

                const outputfile = fs.createWriteStream(outputpath);
                const output = csvstringify({ header: false, delimiter: '\t' });
                output.pipe(outputfile);

                const strings = Array.from(fileContent.file.keys());
                const weights = Array.from(fileContent.file.values());
                const tokenized = this._tokenizeAll(strings);

                for (let i = 0; i < strings.length; i++) {
                    const value = strings[i];
                    const weight = weights[i];
                    const tokens = tokenized[i].tokens;

                    // if some tokens are uppercase, they are entities, like NUMBER_0,
                    // in which case we ignore this value
                    if (tokens.length === 0 || tokens.some((tok) => /^[A-Z]/.test(tok)))
                        continue;

                    const tokenizedString = tokens.join(' ');
                    if (this._maxValueLength >= 0 && tokenizedString.length > this._maxValueLength)
                        continue;

                    output.write([value, tokenizedString, weight]);
                }

                output.end();
                manifest.write(`string\t${this._locale}\torg.wikidata:${fileId}\t${path.relative(manifestDir, outputpath)}\n`);

                //await StreamUtils.waitFinish(output);
                console.log(`completed ${fileId}`);
            }
        }

        manifest.end();
        await StreamUtils.waitFinish(manifest);
    }
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.add_parser('wikidata-make-string-datasets', {
            add_help: true,
            description: "Extract string datasets from a AutoQA normalized data file."
        });
        parser.add_argument('-d', '--output-dir', {
            required: true,
        });
        parser.add_argument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.add_argument('-l', '--locale', {
            required: false,
            default: 'en-US',
            help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
        });
        parser.add_argument('--manifest', {
            required: true,
            help: `Write a parameter dataset manifest to this location`
        });
        parser.add_argument('--append-manifest', {
            required: false,
            action: 'store_true',
            help: `append to the manifest instead of replacing`
        });
        parser.add_argument('--max-value-length', {
            required: false,
            default: 500,
            help: 'Ignore values longer than this (unit: number of UTF-16 code points after tokenization).'
        });
        parser.add_argument('--target-size', {
            required: false,
            default: 1000,
            help: 'target number of examples for each property'
        });
        parser.add_argument('--debug', {
            action: 'store_true',
            help: 'Enable debugging.',
            default: true
        });
        parser.add_argument('--no-debug', {
            action: 'store_false',
            dest: 'debug',
            help: 'Disable debugging.',
        });
    },

    async execute(args) {
        const generator = new ParamDatasetGenerator(args);
        await generator.init(args.thingpedia);
        await generator.run();
        await generator.output(args.output_dir, args.manifest, args.append_manifest);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const assert = require('assert');	51✔
23	const fs = require('fs');	51✔
24	const util = require('util');	51✔
25	const path = require('path');	51✔
26	const ThingTalk = require('thingtalk');	51✔
27	const csvstringify = require('csv-stringify');	51✔
28
29	const I18N = require('../../../lib/i18n');	51✔
30	const StreamUtils = require('../../../lib/utils/stream-utils');	51✔
31
32	const {
33	wikidataQuery,
34	getEquivalent,
35	getItemLabel
36	} = require('./utils');	51✔
37
38	function getElemType(type) {
NEW 39	if (type.isArray)	×
NEW 40	return getElemType(type.elem);	×
NEW 41	return type;	×
42	}
43
44	class ParamDatasetGenerator {
45	constructor(options) {
NEW 46	this._locale = options.locale;	×
NEW 47	this._debug = options.debug;	×
NEW 48	this._maxValueLength = options.max_value_length;	×
NEW 49	this._targetSize = options.target_size;	×
NEW 50	this._stringFiles = new Map;	×
NEW 51	this._tokenizer = I18N.get(options.locale).getTokenizer();	×
52	}
53
54	_getStringFile(stringFileName, isEntity) {
NEW 55	if (this._stringFiles.has(stringFileName))	×
NEW 56	return this._stringFiles.get(stringFileName).file;	×
NEW 57	const file = isEntity ? [] : new Map;	×
NEW 58	this._stringFiles.set(stringFileName, { file, isEntity });	×
NEW 59	return file;	×
60	}
61
62	_addString(stringFileName, value) {
NEW 63	const stringFile = this._getStringFile(stringFileName, false);	×
NEW 64	stringFile.set(value, (stringFile.get(value) \|\| 0) + 1);	×
65	}
66
67	async init(thingpedia) {
NEW 68	const library = ThingTalk.Grammar.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));	×
NEW 69	assert(library.isLibrary && library.classes.length === 1);	×
NEW 70	this._classDef = library.classes[0];	×
71	}
72
73	async _downloadSubjectValues(fn, klass, filters, targetSize) {
NEW 74	const predicate = klass === 'Q5' ? 'wdt:P31' : 'p:P31/ps:P31/wdt:P279*';	×
NEW 75	const query = `	×
76	SELECT DISTINCT ?subject
77	WHERE {
78	?subject ${predicate} wd:${klass}; ${filters.join('; ')}.
79	}
80	LIMIT ${targetSize}
81	`;
NEW 82	const results = await wikidataQuery(query);	×
NEW 83	for (let result of results) {	×
NEW 84	const value = result.subject.value;	×
NEW 85	const label = await getItemLabel(value.slice('http://www.wikidata.org/entity/'.length));	×
NEW 86	if (!label)	×
NEW 87	continue;	×
NEW 88	if (/Q[0-9]+/.test(label))	×
NEW 89	continue;	×
NEW 90	this._getStringFile(fn, true).push({ name: label, value: value });	×
91	}
92
93	}
94
95	async _downloadPropertyValues(fn, arg, klass, filters, targetSize) {
NEW 96	if (arg.name === 'id')	×
NEW 97	return ;	×
NEW 98	const elemType = getElemType(arg.type);	×
NEW 99	if (!elemType.isString && !(elemType.isEntity && elemType.type.startsWith('org.wikidata:')))	×
NEW 100	return;	×
101
NEW 102	const id = arg.getImplementationAnnotation('wikidata_id');	×
103
NEW 104	const predicate = klass === 'Q5' ? 'wdt:P31' : 'p:P31/ps:P31/wdt:P279*';	×
NEW 105	const query = `	×
106	SELECT DISTINCT ?value
107	WHERE {
108	?subject ${predicate} wd:${klass}; ${filters.join('; ')}.
109	?subject wdt:${id} ?value.
110	}
111	LIMIT ${targetSize}
112	`;
NEW 113	const results = await wikidataQuery(query);	×
NEW 114	for (let result of results) {	×
NEW 115	const value = result.value.value;	×
116	let label;
NEW 117	if (value.startsWith('http://www.wikidata.org/entity/'))	×
NEW 118	label = await getItemLabel(value.slice('http://www.wikidata.org/entity/'.length));	×
119	else
NEW 120	label = value;	×
NEW 121	if (!label)	×
NEW 122	continue;	×
NEW 123	if (/Q[0-9]+/.test(label))	×
NEW 124	continue;	×
NEW 125	if (elemType.isString)	×
NEW 126	this._addString(`${fn}_${arg.name}`, label);	×
NEW 127	else if (elemType.isEntity && elemType.type.startsWith('org.wikidata:'))	×
NEW 128	this._getStringFile(elemType.type.slice('org.wikidata:'.length), true).push({ name: label, value: value });	×
129	}
130	}
131
132	async _tryDownloadSubjectValues(fn, klass, triples, targetSize) {
NEW 133	while (targetSize > 30) {	×
NEW 134	try {	×
NEW 135	await this._downloadSubjectValues(fn, klass, triples, Math.ceil(targetSize));	×
NEW 136	return;	×
137	} catch(e) {
NEW 138	if (e.code !== 500)	×
NEW 139	throw e;	×
NEW 140	targetSize /= 2;	×
141	}
142	}
143	}
144
145	async _tryDownloadPropertyValues(fn, arg, klass, triples, targetSize) {
NEW 146	while (targetSize > 30) {	×
NEW 147	try {	×
NEW 148	await this._downloadPropertyValues(fn, arg, klass, triples, Math.ceil(targetSize));	×
NEW 149	return;	×
150	} catch(e) {
NEW 151	if (e.code !== 500)	×
NEW 152	throw e;	×
NEW 153	targetSize /= 2;	×
154	}
155	}
156	}
157
158	async run() {
NEW 159	for (let fn in this._classDef.queries) {	×
NEW 160	const fndef = this._classDef.queries[fn];	×
NEW 161	const klass = fndef.getImplementationAnnotation('wikidata_subject');	×
NEW 162	const equivalentClasses = klass === 'Q5' ? null : await getEquivalent(klass);	×
NEW 163	const classes = equivalentClasses ? [klass, ...equivalentClasses] : [klass];	×
NEW 164	const triples = [];	×
NEW 165	for (let arg of fndef.getImplementationAnnotation('required_properties') \|\| [])	×
NEW 166	triples.push(`wdt:${arg} ?${arg}`);	×
NEW 167	for (let klass of classes) {	×
NEW 168	await this._tryDownloadSubjectValues(fn, klass, triples, Math.ceil(this._targetSize / classes.length));	×
NEW 169	for (let arg of fndef.iterateArguments())	×
NEW 170	await this._tryDownloadPropertyValues(fn, arg, klass, triples, Math.ceil(this._targetSize / classes.length));	×
171	}
172	}
173	}
174
175	_tokenizeAll(strings) {
NEW 176	return strings.map((str) => this._tokenizer.tokenize(str));	×
177	}
178
179	async output(outputDir, manifestFile, appendManifest) {
NEW 180	await util.promisify(fs.mkdir)(outputDir, { recursive: true });	×
181
NEW 182	const manifestDir = path.dirname(manifestFile);	×
NEW 183	const manifest = fs.createWriteStream(manifestFile, { flags: appendManifest ? 'a' : 'w' });	×
184
NEW 185	for (let [fileId, fileContent] of this._stringFiles) {	×
NEW 186	const outputpath = path.resolve(outputDir, 'org.wikidata:' + fileId + (fileContent.isEntity ? '.json' : '.tsv'));	×
187
NEW 188	if (fileContent.isEntity) {	×
NEW 189	if (this._debug)	×
NEW 190	console.log(`Found ${fileContent.file.length} examples for entity file ${fileId}`);	×
191
NEW 192	const tokenized = this._tokenizeAll(fileContent.file.map((entity) => entity.name));	×
NEW 193	const data = [];	×
NEW 194	for (let i = 0; i < fileContent.file.length; i++) {	×
NEW 195	const entity = fileContent.file[i];	×
NEW 196	const tokens = tokenized[i].tokens;	×
197
198	// if some tokens are uppercase, they are entities, like NUMBER_0,
199	// in which case we ignore this value
NEW 200	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
NEW 201	continue;	×
202
NEW 203	entity.canonical = tokens.join(' ');	×
NEW 204	if (this._maxValueLength >= 0 && entity.canonical.length > this._maxValueLength)	×
NEW 205	continue;	×
206
NEW 207	data.push(entity);	×
208	}
209
NEW 210	await util.promisify(fs.writeFile)(outputpath, JSON.stringify({	×
211	result: 'ok', data
212	}, undefined, 2), { encoding: 'utf8' });
213
NEW 214	manifest.write(`entity\t${this._locale}\torg.wikidata:${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
215	} else {
NEW 216	if (this._debug)	×
NEW 217	console.log(`Found ${fileContent.file.size} examples for string file ${fileId}`);	×
218
NEW 219	const outputfile = fs.createWriteStream(outputpath);	×
NEW 220	const output = csvstringify({ header: false, delimiter: '\t' });	×
NEW 221	output.pipe(outputfile);	×
222
NEW 223	const strings = Array.from(fileContent.file.keys());	×
NEW 224	const weights = Array.from(fileContent.file.values());	×
NEW 225	const tokenized = this._tokenizeAll(strings);	×
226
NEW 227	for (let i = 0; i < strings.length; i++) {	×
NEW 228	const value = strings[i];	×
NEW 229	const weight = weights[i];	×
NEW 230	const tokens = tokenized[i].tokens;	×
231
232	// if some tokens are uppercase, they are entities, like NUMBER_0,
233	// in which case we ignore this value
NEW 234	if (tokens.length === 0 \|\| tokens.some((tok) => /^[A-Z]/.test(tok)))	×
NEW 235	continue;	×
236
NEW 237	const tokenizedString = tokens.join(' ');	×
NEW 238	if (this._maxValueLength >= 0 && tokenizedString.length > this._maxValueLength)	×
NEW 239	continue;	×
240
NEW 241	output.write([value, tokenizedString, weight]);	×
242	}
243
NEW 244	output.end();	×
NEW 245	manifest.write(`string\t${this._locale}\torg.wikidata:${fileId}\t${path.relative(manifestDir, outputpath)}\n`);	×
246
247	//await StreamUtils.waitFinish(output);
NEW 248	console.log(`completed ${fileId}`);	×
249	}
250	}
251
NEW 252	manifest.end();	×
NEW 253	await StreamUtils.waitFinish(manifest);	×
254	}
255	}
256
257	module.exports = {	51✔
258	initArgparse(subparsers) {
259	const parser = subparsers.add_parser('wikidata-make-string-datasets', {	51✔
260	add_help: true,
261	description: "Extract string datasets from a AutoQA normalized data file."
262	});
263	parser.add_argument('-d', '--output-dir', {	51✔
264	required: true,
265	});
266	parser.add_argument('--thingpedia', {	51✔
267	required: true,
268	help: 'Path to ThingTalk file containing class definitions.'
269	});
270	parser.add_argument('-l', '--locale', {	51✔
271	required: false,
272	default: 'en-US',
273	help: `BGP 47 locale tag of the language to generate (defaults to 'en-US', English)`
274	});
275	parser.add_argument('--manifest', {	51✔
276	required: true,
277	help: `Write a parameter dataset manifest to this location`
278	});
279	parser.add_argument('--append-manifest', {	51✔
280	required: false,
281	action: 'store_true',
282	help: `append to the manifest instead of replacing`
283	});
284	parser.add_argument('--max-value-length', {	51✔
285	required: false,
286	default: 500,
287	help: 'Ignore values longer than this (unit: number of UTF-16 code points after tokenization).'
288	});
289	parser.add_argument('--target-size', {	51✔
290	required: false,
291	default: 1000,
292	help: 'target number of examples for each property'
293	});
294	parser.add_argument('--debug', {	51✔
295	action: 'store_true',
296	help: 'Enable debugging.',
297	default: true
298	});
299	parser.add_argument('--no-debug', {	51✔
300	action: 'store_false',
301	dest: 'debug',
302	help: 'Disable debugging.',
303	});
304	},
305
306	async execute(args) {
NEW 307	const generator = new ParamDatasetGenerator(args);	×
NEW 308	await generator.init(args.thingpedia);	×
NEW 309	await generator.run();	×
NEW 310	await generator.output(args.output_dir, args.manifest, args.append_manifest);	×
311	}
312	};

stanford-oval / genie-toolkit / 1434

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous