stanford-oval / genie-toolkit / 682

pending completion

Build # 682

Build Type

Pull #144

travis-ci-com

Committed by

web-flow

Commit Message

Add more comment

Pull Request Pull Request #144: More changes from schema2qa

Run Details

1273 of 4686 branches covered (27.17%)

Branch coverage included in aggregate %.

3 of 61 new or added lines in 3 files covered. (4.92%)

6 existing lines in 2 files now uncovered.

3981 of 9492 relevant lines covered (41.94%)

27311.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.52

/tool/lib/webqa-canonical-generator.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Silei Xu <silei@cs.stanford.edu>
//
// See COPYING for details
"use strict";
const fs = require('fs');
const util = require('util');
const path = require('path');
const child_process = require('child_process');
const utils = require('../../lib/utils');

const { makeLookupKeys } = require('../../lib/sample-utils');

const ANNOTATED_PROPERTIES = [
    'url', 'name', 'description',
    'geo', 'address.streetAddress', 'address.addressCountry', 'address.addressRegion', 'address.addressLocality'
];


// extract entity type from type
function typeToEntityType(type) {
    if (type.isArray)
        return typeToEntityType(type.elem);
    else if (type.isEntity)
        return type.type;
    else
        return null;
}

// split the canonical into prefix and suffix
function splitCanonical(canonical) {
    let prefix, suffix;
    if (!canonical.includes('#'))
        [prefix, suffix] = [canonical, ''];
    else if (canonical.includes('#') && !canonical.startsWith('#'))
        [prefix, suffix] = canonical.split('#').map((span) => span.trim());
    else
        [prefix, suffix] = ['', canonical.slice(1).trim()];
    return [prefix, suffix];
}

// return the template query of a certain grammar category
function templateQuery(cat, tableName, prefix, value='', suffix='') {
    switch (cat) {
        case 'base':
            return `what is the ${prefix} of the ${tableName} ?`.split(/\s+/g);
        case 'property':
            return `show me ${tableName} with ${prefix} ${value} ${suffix} .`.split(/\s+/g);
        case 'verb':
            return `which ${tableName} ${prefix} ${value} ${suffix} ?`.split(/\s+/g);
        case 'passive_verb':
            return `show me a ${tableName} ${prefix} ${value} ${suffix} .`.split(/\s+/g);
        case 'reverse_property':
            return `which ${tableName} is a ${prefix} ${value} ${suffix} ?`.split(/\s+/g);
        default:
            throw new Error(`Invalid grammar category ${cat}`);
    }
}


class CanonicalGenerator {
    constructor(classDef, constants, queries, parameterDatasets, options) {
        this.class = classDef;
        this.constants = constants;
        this.queries = queries;

        this.pruning = options.pruning;
        this.mask = options.mask;
        this.is_paraphraser = options.is_paraphraser;
        this.model = options.model;

        this.parameterDatasets = parameterDatasets;
        this.parameterDatasetPaths = {};

        this.sampleSize = {};

        this.options = options;
    }


    async generate() {
        await this._loadParameterDatasetPaths();

        const examples = {};
        const paths = {};
        for (let qname of this.queries) {
            examples[qname] = {};
            paths[qname] = { canonical: this.class.queries[qname].canonical, params: {} };
            let query = this.class.queries[qname];
            let typeCounts = this._getArgTypeCount(qname);
            for (let arg of query.iterateArguments()) {
                if (ANNOTATED_PROPERTIES.includes(arg.name))
                    continue;

                // get the paths to the data
                let p = path.dirname(this.parameterDatasets) + '/'  + this._getDatasetPath(qname, arg);
                if (p && fs.existsSync(p))
                    paths[qname]['params'][`${arg.name}`] = p;

                // some args don't have canonical: e.g., id, name
                if (!arg.metadata.canonical)
                    continue;

                // copy base canonical if property canonical is missing
                if (arg.metadata.canonical.base && !arg.metadata.canonical.property)
                    arg.metadata.canonical.property = [...arg.metadata.canonical.base];

                // if property is missing, try to use entity type info
                if (!('property' in arg.metadata.canonical)) {
                    let typestr = typeToEntityType(query.getArgType(arg.name));
                    // only apply this if there is only one property uses this entity type
                    if (typestr && typeCounts[typestr] === 1) {
                        let base = utils.clean(typestr.substring(typestr.indexOf(':') + 1));
                        arg.metadata.canonical['property'] = [base];
                        arg.metadata.canonical['base'] = [base];
                    }
                }

                const samples = this._retrieveSamples(qname, arg);
                if (samples) {
                    this.sampleSize[`${qname}.${arg.name}`] = samples.length;
                    examples[qname][arg.name] = this._generateExamples(query.canonical, arg.metadata.canonical, samples);
                }
            }
        }

        const args = [path.resolve(path.dirname(module.filename), './bert-canonical-generator.py'), 'all'];
        if (this.is_paraphraser)
            args.push('--is-paraphraser');
        args.push('--model-name-or-path');
        args.push(this.model);
        args.push(this.mask ? '--mask' : '--no-mask');

        // call bert to generate candidates
        const child = child_process.spawn(`python3`, args, { stdio: ['pipe', 'pipe', 'inherit'] });

        const output = util.promisify(fs.writeFile);
        if (this.options.debug)
            await output(`./bert-annotator-in.json`, JSON.stringify({ examples, paths }, null, 2));

        const stdout = await new Promise((resolve, reject) => {
            child.stdin.write(JSON.stringify( { examples, paths } ));
            child.stdin.end();
            child.on('error', reject);
            child.stdout.on('error', reject);
            child.stdout.setEncoding('utf8');
            let buffer = '';
            child.stdout.on('data', (data) => {
                buffer += data;
            });
            child.stdout.on('end', () => resolve(buffer));
        });

        if (this.options.debug)
            await output(`./bert-annotator-out.json`, JSON.stringify(JSON.parse(stdout), null, 2));

        const { synonyms, adjectives } = JSON.parse(stdout);
        this._updateCanonicals(synonyms, adjectives);
        return this.class;
    }

    _getArgTypeCount(qname) {
        const schema = this.class.queries[qname];
        const count = {};
        for (let arg of schema.iterateArguments()) {
            let typestr = typeToEntityType(schema.getArgType(arg.name));
            if (!typestr)
                continue;
            count[typestr] = (count[typestr] || 0) + 1;
        }
        return count;
    }

    async _loadParameterDatasetPaths() {
        const rows = (await (util.promisify(fs.readFile))(this.parameterDatasets, { encoding: 'utf8' })).split('\n');
        for (let row of rows) {
            let [, key, path] = row.split('\t');
            this.parameterDatasetPaths[key] = path;
        }
    }

    _getDatasetPath(qname, arg) {
        const keys = [
            `${this.class.kind}:${qname}_${arg.name}`,
            `${arg.type.isEntity ? arg.type.type : arg.type}`
        ];
        for (let key of keys) {
            if (this.parameterDatasetPaths[key])
                return this.parameterDatasetPaths[key];
        }
        return null;
    }

    _updateCanonicals(candidates, adjectives) {
        for (let qname of this.queries) {
            for (let arg in candidates[qname]) {
                let canonicals = this.class.queries[qname].getArgument(arg).metadata.canonical;
                if (adjectives.includes(`${qname}.${arg}`))
                        canonicals['adjective'] = ['#'];

                for (let type in candidates[qname][arg]) {
                    let count = candidates[qname][arg][type].candidates;
                    let max = candidates[qname][arg][type].examples.filter((e) => e.candidates.length > 0).length;
                    for (let candidate in count) {
                        if (count[candidate] > max * this.pruning) {
                            if (!canonicals[type].includes(candidate))
                                canonicals[type].push(candidate);
                        }
                    }

                }
            }
        }
    }

    _retrieveSamples(qname, arg) {
        const keys = makeLookupKeys('@' + this.class.kind + '.' + qname, arg.name, arg.type);
        let samples;
        for (let key of keys) {
            if (this.constants[key]) {
                samples = this.constants[key];
                break;
            }
        }
        if (samples) {
            samples = samples.map((v) => {
                if (arg.type.isString)
                    return v.value;
                return v.display;
            });
        }
        return samples;
    }

    _generateExamples(tableName, canonicals, valueSample) {
        let examples = {};
        for (let cat of ['base', 'property', 'verb', 'passive_verb', 'reverse_property']) {
            if (cat in canonicals)
                examples[cat] = { examples: [], candidates: [] } ;
        }

        if ('base' in canonicals) {
            for (let canonical of canonicals['base']) {
                let query = templateQuery('base', tableName, canonical);
                let maskIndices = canonical.split(' ').map((w) => query.indexOf(w));
                examples['base']['examples'].push({
                    query: query.join(' '),
                    masks: {prefix: maskIndices, suffix: []},
                    value: []
                });
            }
        }

        for (let value of valueSample) {
            for (let cat in canonicals) {
                if (['default', 'adjective', 'implicit_identity', 'base'].includes(cat))
                    continue;
                for (let canonical of canonicals[cat]) {
                    let [prefix, suffix] = splitCanonical(canonical);
                    let query = templateQuery(cat, tableName, prefix, value, suffix);
                    let prefixIndices = prefix ? prefix.split(' ').map((w) => query.indexOf(w)) : [];
                    let suffixIndices = suffix ? suffix.split(' ').map((w) => query.indexOf(w)) : [];
                    let valueIndices = value.split(' ').map((w) => query.indexOf(w));
                    examples[cat]['examples'].push({
                        query: query.join(' '),
                        masks: { prefix: prefixIndices, suffix: suffixIndices },
                        value: valueIndices
                    });
                }
            }
        }
        return examples;
    }
}

module.exports = CanonicalGenerator;

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Silei Xu <silei@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11	const fs = require('fs');	46✔
12	const util = require('util');	46✔
13	const path = require('path');	46✔
14	const child_process = require('child_process');	46✔
15	const utils = require('../../lib/utils');	46✔
16
17	const { makeLookupKeys } = require('../../lib/sample-utils');	46✔
18
19	const ANNOTATED_PROPERTIES = [	46✔
20	'url', 'name', 'description',
21	'geo', 'address.streetAddress', 'address.addressCountry', 'address.addressRegion', 'address.addressLocality'
22	];
23
24
25	// extract entity type from type
26	function typeToEntityType(type) {
NEW 27	if (type.isArray)	×
NEW 28	return typeToEntityType(type.elem);	×
NEW 29	else if (type.isEntity)	×
NEW 30	return type.type;	×
31	else
NEW 32	return null;	×
33	}
34
35	// split the canonical into prefix and suffix
36	function splitCanonical(canonical) {
37	let prefix, suffix;
38	if (!canonical.includes('#'))	×
39	[prefix, suffix] = [canonical, ''];	×
40	else if (canonical.includes('#') && !canonical.startsWith('#'))	×
41	[prefix, suffix] = canonical.split('#').map((span) => span.trim());	×
42	else
43	[prefix, suffix] = ['', canonical.slice(1).trim()];	×
44	return [prefix, suffix];	×
45	}
46
47	// return the template query of a certain grammar category
48	function templateQuery(cat, tableName, prefix, value='', suffix='') {	×
49	switch (cat) {	×
50	case 'base':
51	return `what is the ${prefix} of the ${tableName} ?`.split(/\s+/g);	×
52	case 'property':
UNCOV 53	return `show me ${tableName} with ${prefix} ${value} ${suffix} .`.split(/\s+/g);	×
54	case 'verb':
UNCOV 55	return `which ${tableName} ${prefix} ${value} ${suffix} ?`.split(/\s+/g);	×
56	case 'passive_verb':
UNCOV 57	return `show me a ${tableName} ${prefix} ${value} ${suffix} .`.split(/\s+/g);	×
58	case 'reverse_property':
UNCOV 59	return `which ${tableName} is a ${prefix} ${value} ${suffix} ?`.split(/\s+/g);	×
60	default:
61	throw new Error(`Invalid grammar category ${cat}`);	×
62	}
63	}
64
65
66	class CanonicalGenerator {
67	constructor(classDef, constants, queries, parameterDatasets, options) {
68	this.class = classDef;	×
69	this.constants = constants;	×
70	this.queries = queries;	×
71
72	this.pruning = options.pruning;	×
73	this.mask = options.mask;	×
74	this.is_paraphraser = options.is_paraphraser;	×
75	this.model = options.model;	×
76
77	this.parameterDatasets = parameterDatasets;	×
78	this.parameterDatasetPaths = {};	×
79
80	this.sampleSize = {};	×
81
82	this.options = options;	×
83	}
84
85
86	async generate() {
87	await this._loadParameterDatasetPaths();	×
88
89	const examples = {};	×
90	const paths = {};	×
91	for (let qname of this.queries) {	×
92	examples[qname] = {};	×
93	paths[qname] = { canonical: this.class.queries[qname].canonical, params: {} };	×
94	let query = this.class.queries[qname];	×
NEW 95	let typeCounts = this._getArgTypeCount(qname);	×
96	for (let arg of query.iterateArguments()) {	×
97	if (ANNOTATED_PROPERTIES.includes(arg.name))	×
98	continue;	×
99
100	// get the paths to the data
101	let p = path.dirname(this.parameterDatasets) + '/' + this._getDatasetPath(qname, arg);	×
102	if (p && fs.existsSync(p))	×
103	paths[qname]['params'][`${arg.name}`] = p;	×
104
105	// some args don't have canonical: e.g., id, name
106	if (!arg.metadata.canonical)	×
107	continue;	×
108
109	// copy base canonical if property canonical is missing
NEW 110	if (arg.metadata.canonical.base && !arg.metadata.canonical.property)	×
NEW 111	arg.metadata.canonical.property = [...arg.metadata.canonical.base];	×
112
113	// if property is missing, try to use entity type info
NEW 114	if (!('property' in arg.metadata.canonical)) {	×
NEW 115	let typestr = typeToEntityType(query.getArgType(arg.name));	×
116	// only apply this if there is only one property uses this entity type
NEW 117	if (typestr && typeCounts[typestr] === 1) {	×
NEW 118	let base = utils.clean(typestr.substring(typestr.indexOf(':') + 1));	×
NEW 119	arg.metadata.canonical['property'] = [base];	×
NEW 120	arg.metadata.canonical['base'] = [base];	×
121	}
122	}
123
124	const samples = this._retrieveSamples(qname, arg);	×
125	if (samples) {	×
126	this.sampleSize[`${qname}.${arg.name}`] = samples.length;	×
127	examples[qname][arg.name] = this._generateExamples(query.canonical, arg.metadata.canonical, samples);	×
128	}
129	}
130	}
131
132	const args = [path.resolve(path.dirname(module.filename), './bert-canonical-generator.py'), 'all'];	×
133	if (this.is_paraphraser)	×
134	args.push('--is-paraphraser');	×
135	args.push('--model-name-or-path');	×
136	args.push(this.model);	×
137	args.push(this.mask ? '--mask' : '--no-mask');	×
138
139	// call bert to generate candidates
140	const child = child_process.spawn(`python3`, args, { stdio: ['pipe', 'pipe', 'inherit'] });	×
141
142	const output = util.promisify(fs.writeFile);	×
143	if (this.options.debug)	×
144	await output(`./bert-annotator-in.json`, JSON.stringify({ examples, paths }, null, 2));	×
145
146	const stdout = await new Promise((resolve, reject) => {	×
147	child.stdin.write(JSON.stringify( { examples, paths } ));	×
148	child.stdin.end();	×
149	child.on('error', reject);	×
150	child.stdout.on('error', reject);	×
151	child.stdout.setEncoding('utf8');	×
152	let buffer = '';	×
153	child.stdout.on('data', (data) => {	×
154	buffer += data;	×
155	});
156	child.stdout.on('end', () => resolve(buffer));	×
157	});
158
159	if (this.options.debug)	×
160	await output(`./bert-annotator-out.json`, JSON.stringify(JSON.parse(stdout), null, 2));	×
161
162	const { synonyms, adjectives } = JSON.parse(stdout);	×
163	this._updateCanonicals(synonyms, adjectives);	×
164	return this.class;	×
165	}
166
167	_getArgTypeCount(qname) {
NEW 168	const schema = this.class.queries[qname];	×
NEW 169	const count = {};	×
NEW 170	for (let arg of schema.iterateArguments()) {	×
NEW 171	let typestr = typeToEntityType(schema.getArgType(arg.name));	×
NEW 172	if (!typestr)	×
NEW 173	continue;	×
NEW 174	count[typestr] = (count[typestr] \|\| 0) + 1;	×
175	}
NEW 176	return count;	×
177	}
178
179	async _loadParameterDatasetPaths() {
180	const rows = (await (util.promisify(fs.readFile))(this.parameterDatasets, { encoding: 'utf8' })).split('\n');	×
181	for (let row of rows) {	×
182	let [, key, path] = row.split('\t');	×
183	this.parameterDatasetPaths[key] = path;	×
184	}
185	}
186
187	_getDatasetPath(qname, arg) {
188	const keys = [	×
189	`${this.class.kind}:${qname}_${arg.name}`,
190	`${arg.type.isEntity ? arg.type.type : arg.type}`	×
191	];
192	for (let key of keys) {	×
193	if (this.parameterDatasetPaths[key])	×
194	return this.parameterDatasetPaths[key];	×
195	}
196	return null;	×
197	}
198
199	_updateCanonicals(candidates, adjectives) {
200	for (let qname of this.queries) {	×
201	for (let arg in candidates[qname]) {	×
202	let canonicals = this.class.queries[qname].getArgument(arg).metadata.canonical;	×
203	if (adjectives.includes(`${qname}.${arg}`))	×
NEW 204	canonicals['adjective'] = ['#'];	×
205
206	for (let type in candidates[qname][arg]) {	×
207	let count = candidates[qname][arg][type].candidates;	×
208	let max = candidates[qname][arg][type].examples.filter((e) => e.candidates.length > 0).length;	×
209	for (let candidate in count) {	×
210	if (count[candidate] > max * this.pruning) {	×
211	if (!canonicals[type].includes(candidate))	×
212	canonicals[type].push(candidate);	×
213	}
214	}
215
216	}
217	}
218	}
219	}
220
221	_retrieveSamples(qname, arg) {
222	const keys = makeLookupKeys('@' + this.class.kind + '.' + qname, arg.name, arg.type);	×
223	let samples;
224	for (let key of keys) {	×
225	if (this.constants[key]) {	×
226	samples = this.constants[key];	×
227	break;	×
228	}
229	}
230	if (samples) {	×
231	samples = samples.map((v) => {	×
232	if (arg.type.isString)	×
233	return v.value;	×
234	return v.display;	×
235	});
236	}
237	return samples;	×
238	}
239
240	_generateExamples(tableName, canonicals, valueSample) {
241	let examples = {};	×
NEW 242	for (let cat of ['base', 'property', 'verb', 'passive_verb', 'reverse_property']) {	×
243	if (cat in canonicals)	×
244	examples[cat] = { examples: [], candidates: [] } ;	×
245	}
246
247	if ('base' in canonicals) {	×
248	for (let canonical of canonicals['base']) {	×
249	let query = templateQuery('base', tableName, canonical);	×
250	let maskIndices = canonical.split(' ').map((w) => query.indexOf(w));	×
251	examples['base']['examples'].push({	×
252	query: query.join(' '),
253	masks: {prefix: maskIndices, suffix: []},
254	value: []
255	});
256	}
257	}
258
259	for (let value of valueSample) {	×
260	for (let cat in canonicals) {	×
NEW 261	if (['default', 'adjective', 'implicit_identity', 'base'].includes(cat))	×
262	continue;	×
263	for (let canonical of canonicals[cat]) {	×
264	let [prefix, suffix] = splitCanonical(canonical);	×
265	let query = templateQuery(cat, tableName, prefix, value, suffix);	×
266	let prefixIndices = prefix ? prefix.split(' ').map((w) => query.indexOf(w)) : [];	×
267	let suffixIndices = suffix ? suffix.split(' ').map((w) => query.indexOf(w)) : [];	×
268	let valueIndices = value.split(' ').map((w) => query.indexOf(w));	×
269	examples[cat]['examples'].push({	×
270	query: query.join(' '),
271	masks: { prefix: prefixIndices, suffix: suffixIndices },
272	value: valueIndices
273	});
274	}
275	}
276	}
277	return examples;	×
278	}
279	}
280
281	module.exports = CanonicalGenerator;	46✔

stanford-oval / genie-toolkit / 682

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous