stanford-oval / genie-toolkit / 812

pending completion

Build # 812

Build Type

Pull #172

travis-ci-com

Committed by

web-flow

Commit Message

Merge 86aa0a447 into 6c6a00144

Pull Request Pull Request #172: Wip/schemaorg automation

Run Details

1347 of 5156 branches covered (26.12%)

Branch coverage included in aggregate %.

32 of 326 new or added lines in 9 files covered. (9.82%)

8 existing lines in 3 files now uncovered.

4125 of 10108 relevant lines covered (40.81%)

16352.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

2.3

/tool/lib/webqa-template-extractor.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Silei Xu <silei@cs.stanford.edu>
//
// See COPYING for details
"use strict";
const fs = require('fs');
const util = require('util');
const POS = require("en-pos");
const child_process = require('child_process');

function posTag(tokens) {
    return new POS.Tag(tokens)
        .initial() // initial dictionary and pattern based tagging
        .smooth() // further context based smoothing
        .tags;
}

class AnnotationExtractor {
    constructor(klass, queries, model, options) {
        this.class = klass;
        this.model = model;
        this.queries = queries;
        this.options = options;

        this.newCanonicals = {};
    }

    async run(synonyms, queries) {
        for (let qname of this.queries) {
            let query_canonical = queries[qname]['canonical'];
            for (let arg in synonyms[qname]) {
                if (arg === 'id' || Object.keys(synonyms[qname][arg]).length === 0)
                    continue;

                let input = this.generateInput(synonyms[qname][arg]);
                let output = await this._paraphrase(input.join('\n'), arg);
                let values = queries[qname]['args'][arg]['values'];
                for (let i = 0; i < input.length; i++)
                    this.extractCanonical(arg, input[i], output[i], values, query_canonical);
            }

            for (let arg in synonyms[qname]) {
                if (!(arg in this.newCanonicals))
                    continue;

                let canonicals = this.class.queries[qname].getArgument(arg).metadata.canonical;
                for (let typeNewCanonical in this.newCanonicals[arg]) {
                    for (let newCanonical of this.newCanonicals[arg][typeNewCanonical]) {
                        if (!(newCanonical.startsWith('# ') || newCanonical.endsWith(' #') || newCanonical.includes(' # ')))
                            continue;

                        if (this.hasConflict(arg, newCanonical))
                            continue;

                        if (!canonicals[typeNewCanonical]) {
                            canonicals[typeNewCanonical] = [newCanonical];
                            continue;
                        }
                        if (canonicals[typeNewCanonical].includes(newCanonical))
                            continue;
                        if (newCanonical.endsWith(' #') && canonicals[typeNewCanonical].includes(newCanonical.slice(0, -2)))
                            continue;
                        canonicals[typeNewCanonical].push(newCanonical);
                    }
                }
            }
        }
    }

    hasConflict(currentArg, currentCanonical) {
        for (let arg in this.newCanonicals) {
            if (arg === currentArg)
                continue;
            for (let pos in this.newCanonicals[arg]) {
                for (let canonical of this.newCanonicals[arg][pos]) {
                    if (canonical.replace('#', '').trim() === currentCanonical.replace('#', '').trim())
                        return true;
                }
            }
        }
        return false;
    }

    async _paraphrase(input, arg) {
        // if debug file exists, use them directly
        if (fs.existsSync(`./gpt2-paraphraser-out-${arg}.json`))
            return JSON.parse(fs.readFileSync(`./gpt2-paraphraser-out-${arg}.json`, 'utf-8'));

        // genienlp run-paraphrase --input_column 0 --skip_heuristics --model_name_or_path xxx --temperature 1 1 1 --num_beams 4 --pipe_mode
        const args = [
            `run-paraphrase`,
            `--input_column`, `0`,
            `--skip_heuristics`,
            `--model_name_or_path`, this.model,
            `--temperature`, `1`, `1`, `1`,
            `--num_beams`, `4`,
            `--pipe_mode`
        ];
        const child = child_process.spawn(`genienlp`, args, { stdio: ['pipe', 'pipe', 'inherit'] });

        const output = util.promisify(fs.writeFile);
        if (this.options.debug)
            await output(`./gpt2-paraphraser-in-${arg}.tsv`, input);

        const stdout = await new Promise((resolve, reject) => {
            child.stdin.write(input);
            child.stdin.end();
            child.on('error', reject);
            child.stdout.on('error', reject);
            child.stdout.setEncoding('utf8');
            let buffer = '';
            child.stdout.on('data', (data) => {
                buffer += data;
            });
            child.stdout.on('end', () => resolve(buffer));
        });

        if (this.options.debug)
            await output(`./gpt2-paraphraser-out-${arg}.json`, JSON.stringify(JSON.parse(stdout), null, 2));

        return JSON.parse(stdout);
    }

    generateInput(candidates) {
        const input = [];
        for (let category in candidates) {
            if (category === 'base')
                continue;
            let canonical = Object.keys(candidates[category])[0];
            for (let sentence of candidates[category][canonical])
                input.push(`${sentence}`);
        }
        return input;
    }

    extractCanonical(arg, origin, paraphrases, values, query_canonical) {
        if (!(arg in this.newCanonicals))
            this.newCanonicals[arg] = {};

        const canonical = this.newCanonicals[arg];
        let value = values.find((v) => origin.includes(v));
        if (!value) {
            // base canonical, do nothing
            return;
        }
        value = value.toLowerCase();

        for (let paraphrase of paraphrases) {
            paraphrase = paraphrase.toLowerCase();

            if (!paraphrase.includes(value))
                continue;

            if (paraphrase.endsWith('.') || paraphrase.endsWith('?') || paraphrase.endsWith('!'))
                paraphrase = paraphrase.slice(0, -1);

            let tags = posTag(paraphrase.split(' '));

            let prefixes = [];
            if (origin.startsWith('who ')) {
                prefixes.push('who ');
                prefixes.push('who\'s ');
            } else {
                let standard_prefix = origin.slice(0, origin.indexOf(query_canonical) + query_canonical.length + 1);
                prefixes.push(standard_prefix);
                let to_replace = origin.includes(`a ${query_canonical}`) ? `a ${query_canonical}` : query_canonical;
                prefixes.push(standard_prefix.replace(to_replace, `${query_canonical}s`));
                prefixes.push(standard_prefix.replace(to_replace, `some ${query_canonical}s`));
                prefixes.push(standard_prefix.replace(to_replace, `all ${query_canonical}s`));
                prefixes.push(standard_prefix.replace(to_replace, `any ${query_canonical}s`));
                prefixes.push(standard_prefix.replace(to_replace, `any ${query_canonical}`));
                prefixes.push(standard_prefix.replace(to_replace, `an ${query_canonical}`));
                prefixes.push(standard_prefix.replace(to_replace, `the ${query_canonical}`));
            }

            for (let prefix of new Set(prefixes)) {
                if (!paraphrase.startsWith(prefix))
                    continue;

                let clause = paraphrase.slice(prefix.length);
                let length = prefix.trim().split(' ').length;

                if (prefix === 'who\'s' || clause.startsWith('is ') || clause.startsWith('are ')) {
                    if (clause.startsWith('is ') || clause.startsWith('are ')) {
                        clause = clause.slice(clause.indexOf(' ') + 1);
                        length += 1;
                    }
                    if (clause.startsWith('a ') || clause.startsWith('an ') || clause.startsWith('the ') ||
                        ['NN', 'NNS', 'NNP', 'NNPS'].includes(tags[length + 1])) {
                        canonical['reverse_property'] = canonical['reverse_property'] || [];
                        canonical['reverse_property'].push(clause.replace(value, '#'));
                    } else if (['IN', 'VBN', 'VBG'].includes(tags[length + 1])) {
                        canonical['passive_verb'] = canonical['passive_verb'] || [];
                        canonical['passive_verb'].push(clause.replace(value, '#'));
                    }
                } if (clause.startsWith('with ') || clause.startsWith('has ') || clause.startsWith('have ')) {
                    canonical['property'] = canonical['property'] || [];
                    canonical['property'].push(clause.slice(clause.indexOf(' ') + 1).replace(value, '#'));
                } else if ((clause.startsWith('that ') || clause.startsWith('who ')) && ['VBP', 'VBZ', 'VBD'].includes(tags[length + 1])) {
                    canonical['verb'] = canonical['verb'] || [];
                    canonical['verb'].push(clause.slice(clause.indexOf(' ' + 1)).replace(value, '#'));
                } else if (['VBP', 'VBZ', 'VBD'].includes(tags[length])) {
                    canonical['verb'] = canonical['verb'] || [];
                    canonical['verb'].push(clause.replace(value, '#'));
                }
                break;
            }
        }
    }
}


module.exports = AnnotationExtractor;

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Silei Xu <silei@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11	const fs = require('fs');	52✔
12	const util = require('util');	52✔
13	const POS = require("en-pos");	52✔
14	const child_process = require('child_process');	52✔
15
16	function posTag(tokens) {
NEW 17	return new POS.Tag(tokens)	×
18	.initial() // initial dictionary and pattern based tagging
19	.smooth() // further context based smoothing
20	.tags;
21	}
22
23	class AnnotationExtractor {
24	constructor(klass, queries, model, options) {
NEW 25	this.class = klass;	×
NEW 26	this.model = model;	×
NEW 27	this.queries = queries;	×
NEW 28	this.options = options;	×
29
NEW 30	this.newCanonicals = {};	×
31	}
32
33	async run(synonyms, queries) {
NEW 34	for (let qname of this.queries) {	×
NEW 35	let query_canonical = queries[qname]['canonical'];	×
NEW 36	for (let arg in synonyms[qname]) {	×
NEW 37	if (arg === 'id' \|\| Object.keys(synonyms[qname][arg]).length === 0)	×
NEW 38	continue;	×
39
NEW 40	let input = this.generateInput(synonyms[qname][arg]);	×
NEW 41	let output = await this._paraphrase(input.join('\n'), arg);	×
NEW 42	let values = queries[qname]['args'][arg]['values'];	×
NEW 43	for (let i = 0; i < input.length; i++)	×
NEW 44	this.extractCanonical(arg, input[i], output[i], values, query_canonical);	×
45	}
46
NEW 47	for (let arg in synonyms[qname]) {	×
NEW 48	if (!(arg in this.newCanonicals))	×
NEW 49	continue;	×
50
NEW 51	let canonicals = this.class.queries[qname].getArgument(arg).metadata.canonical;	×
NEW 52	for (let typeNewCanonical in this.newCanonicals[arg]) {	×
NEW 53	for (let newCanonical of this.newCanonicals[arg][typeNewCanonical]) {	×
NEW 54	if (!(newCanonical.startsWith('# ') \|\| newCanonical.endsWith(' #') \|\| newCanonical.includes(' # ')))	×
NEW 55	continue;	×
56
NEW 57	if (this.hasConflict(arg, newCanonical))	×
NEW 58	continue;	×
59
NEW 60	if (!canonicals[typeNewCanonical]) {	×
NEW 61	canonicals[typeNewCanonical] = [newCanonical];	×
NEW 62	continue;	×
63	}
NEW 64	if (canonicals[typeNewCanonical].includes(newCanonical))	×
NEW 65	continue;	×
NEW 66	if (newCanonical.endsWith(' #') && canonicals[typeNewCanonical].includes(newCanonical.slice(0, -2)))	×
NEW 67	continue;	×
NEW 68	canonicals[typeNewCanonical].push(newCanonical);	×
69	}
70	}
71	}
72	}
73	}
74
75	hasConflict(currentArg, currentCanonical) {
NEW 76	for (let arg in this.newCanonicals) {	×
NEW 77	if (arg === currentArg)	×
NEW 78	continue;	×
NEW 79	for (let pos in this.newCanonicals[arg]) {	×
NEW 80	for (let canonical of this.newCanonicals[arg][pos]) {	×
NEW 81	if (canonical.replace('#', '').trim() === currentCanonical.replace('#', '').trim())	×
NEW 82	return true;	×
83	}
84	}
85	}
NEW 86	return false;	×
87	}
88
89	async _paraphrase(input, arg) {
90	// if debug file exists, use them directly
NEW 91	if (fs.existsSync(`./gpt2-paraphraser-out-${arg}.json`))	×
NEW 92	return JSON.parse(fs.readFileSync(`./gpt2-paraphraser-out-${arg}.json`, 'utf-8'));	×
93
94	// genienlp run-paraphrase --input_column 0 --skip_heuristics --model_name_or_path xxx --temperature 1 1 1 --num_beams 4 --pipe_mode
NEW 95	const args = [	×
96	`run-paraphrase`,
97	`--input_column`, `0`,
98	`--skip_heuristics`,
99	`--model_name_or_path`, this.model,
100	`--temperature`, `1`, `1`, `1`,
101	`--num_beams`, `4`,
102	`--pipe_mode`
103	];
NEW 104	const child = child_process.spawn(`genienlp`, args, { stdio: ['pipe', 'pipe', 'inherit'] });	×
105
NEW 106	const output = util.promisify(fs.writeFile);	×
NEW 107	if (this.options.debug)	×
NEW 108	await output(`./gpt2-paraphraser-in-${arg}.tsv`, input);	×
109
NEW 110	const stdout = await new Promise((resolve, reject) => {	×
NEW 111	child.stdin.write(input);	×
NEW 112	child.stdin.end();	×
NEW 113	child.on('error', reject);	×
NEW 114	child.stdout.on('error', reject);	×
NEW 115	child.stdout.setEncoding('utf8');	×
NEW 116	let buffer = '';	×
NEW 117	child.stdout.on('data', (data) => {	×
NEW 118	buffer += data;	×
119	});
NEW 120	child.stdout.on('end', () => resolve(buffer));	×
121	});
122
NEW 123	if (this.options.debug)	×
NEW 124	await output(`./gpt2-paraphraser-out-${arg}.json`, JSON.stringify(JSON.parse(stdout), null, 2));	×
125
NEW 126	return JSON.parse(stdout);	×
127	}
128
129	generateInput(candidates) {
NEW 130	const input = [];	×
NEW 131	for (let category in candidates) {	×
NEW 132	if (category === 'base')	×
NEW 133	continue;	×
NEW 134	let canonical = Object.keys(candidates[category])[0];	×
NEW 135	for (let sentence of candidates[category][canonical])	×
NEW 136	input.push(`${sentence}`);	×
137	}
NEW 138	return input;	×
139	}
140
141	extractCanonical(arg, origin, paraphrases, values, query_canonical) {
NEW 142	if (!(arg in this.newCanonicals))	×
NEW 143	this.newCanonicals[arg] = {};	×
144
NEW 145	const canonical = this.newCanonicals[arg];	×
NEW 146	let value = values.find((v) => origin.includes(v));	×
NEW 147	if (!value) {	×
148	// base canonical, do nothing
NEW 149	return;	×
150	}
NEW 151	value = value.toLowerCase();	×
152
NEW 153	for (let paraphrase of paraphrases) {	×
NEW 154	paraphrase = paraphrase.toLowerCase();	×
155
NEW 156	if (!paraphrase.includes(value))	×
NEW 157	continue;	×
158
NEW 159	if (paraphrase.endsWith('.') \|\| paraphrase.endsWith('?') \|\| paraphrase.endsWith('!'))	×
NEW 160	paraphrase = paraphrase.slice(0, -1);	×
161
NEW 162	let tags = posTag(paraphrase.split(' '));	×
163
NEW 164	let prefixes = [];	×
NEW 165	if (origin.startsWith('who ')) {	×
NEW 166	prefixes.push('who ');	×
NEW 167	prefixes.push('who\'s ');	×
168	} else {
NEW 169	let standard_prefix = origin.slice(0, origin.indexOf(query_canonical) + query_canonical.length + 1);	×
NEW 170	prefixes.push(standard_prefix);	×
NEW 171	let to_replace = origin.includes(`a ${query_canonical}`) ? `a ${query_canonical}` : query_canonical;	×
NEW 172	prefixes.push(standard_prefix.replace(to_replace, `${query_canonical}s`));	×
NEW 173	prefixes.push(standard_prefix.replace(to_replace, `some ${query_canonical}s`));	×
NEW 174	prefixes.push(standard_prefix.replace(to_replace, `all ${query_canonical}s`));	×
NEW 175	prefixes.push(standard_prefix.replace(to_replace, `any ${query_canonical}s`));	×
NEW 176	prefixes.push(standard_prefix.replace(to_replace, `any ${query_canonical}`));	×
NEW 177	prefixes.push(standard_prefix.replace(to_replace, `an ${query_canonical}`));	×
NEW 178	prefixes.push(standard_prefix.replace(to_replace, `the ${query_canonical}`));	×
179	}
180
NEW 181	for (let prefix of new Set(prefixes)) {	×
NEW 182	if (!paraphrase.startsWith(prefix))	×
NEW 183	continue;	×
184
NEW 185	let clause = paraphrase.slice(prefix.length);	×
NEW 186	let length = prefix.trim().split(' ').length;	×
187
NEW 188	if (prefix === 'who\'s' \|\| clause.startsWith('is ') \|\| clause.startsWith('are ')) {	×
NEW 189	if (clause.startsWith('is ') \|\| clause.startsWith('are ')) {	×
NEW 190	clause = clause.slice(clause.indexOf(' ') + 1);	×
NEW 191	length += 1;	×
192	}
NEW 193	if (clause.startsWith('a ') \|\| clause.startsWith('an ') \|\| clause.startsWith('the ') \|\|	×
194	['NN', 'NNS', 'NNP', 'NNPS'].includes(tags[length + 1])) {
NEW 195	canonical['reverse_property'] = canonical['reverse_property'] \|\| [];	×
NEW 196	canonical['reverse_property'].push(clause.replace(value, '#'));	×
NEW 197	} else if (['IN', 'VBN', 'VBG'].includes(tags[length + 1])) {	×
NEW 198	canonical['passive_verb'] = canonical['passive_verb'] \|\| [];	×
NEW 199	canonical['passive_verb'].push(clause.replace(value, '#'));	×
200	}
NEW 201	} if (clause.startsWith('with ') \|\| clause.startsWith('has ') \|\| clause.startsWith('have ')) {	×
NEW 202	canonical['property'] = canonical['property'] \|\| [];	×
NEW 203	canonical['property'].push(clause.slice(clause.indexOf(' ') + 1).replace(value, '#'));	×
NEW 204	} else if ((clause.startsWith('that ') \|\| clause.startsWith('who ')) && ['VBP', 'VBZ', 'VBD'].includes(tags[length + 1])) {	×
NEW 205	canonical['verb'] = canonical['verb'] \|\| [];	×
NEW 206	canonical['verb'].push(clause.slice(clause.indexOf(' ' + 1)).replace(value, '#'));	×
NEW 207	} else if (['VBP', 'VBZ', 'VBD'].includes(tags[length])) {	×
NEW 208	canonical['verb'] = canonical['verb'] \|\| [];	×
NEW 209	canonical['verb'].push(clause.replace(value, '#'));	×
210	}
NEW 211	break;	×
212	}
213	}
214	}
215	}
216
217
218	module.exports = AnnotationExtractor;	52✔

stanford-oval / genie-toolkit / 812

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous