1815

pending completion

Build # 1815

Build Type

Pull #527

travis-ci-com

Committed by

web-flow

Commit Message

Merge 9c23c4529 into 7835066ad

Pull Request Pull Request #527: Wikidata

Run Details

15525 of 40897 branches covered (37.96%)

Branch coverage included in aggregate %.

62 of 509 new or added lines in 5 files covered. (12.18%)

121 existing lines in 5 files now uncovered.

32670 of 72727 relevant lines covered (44.92%)

303947.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.36

/tool/autoqa/auto-annotate.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>


import * as fs from 'fs';
import assert from 'assert';
import util from 'util';
import * as ThingTalk from 'thingtalk';

import * as StreamUtils from '../../lib/utils/stream-utils';

import { parseConstantFile } from '../lib/constant-file';

import AnnotationGenerator from './lib/annotation-generator';

async function loadClassDefs(thingpedia) {
    const library = ThingTalk.Syntax.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));
    assert(library instanceof ThingTalk.Ast.Library);
    return library.classes;
}

export function initArgparse(subparsers) {
    const parser = subparsers.add_parser('auto-annotate', {
        add_help: true,
        description: "Automatically generate annotations including canonicals"
    });
    parser.add_argument('--dataset', {
        required: true,
        choices: ['schemaorg', 'sgd', 'multiwoz', 'wikidata', 'custom'],
        help: 'The dataset to run autoQA on.'
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('-l', '--locale', {
        default: 'en-US',
        help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
    });
    parser.add_argument('--constants', {
        required: true,
        help: 'TSV file containing constant values to use.'
    });
    parser.add_argument('--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('--functions', {
        required: false,
        default: null,
        help: `List of functions to include, split by comma (no space). Include all functions if not specified`,
    });
    parser.add_argument('--remove-existing-canonicals', {
        action: 'store_true',
        help: 'Remove all existing canonical annotations in the schema',
        default: false
    });
    parser.add_argument('--algorithms', {
        nargs: '*',
        help: 'Different algorithms to generate canonicals including bert-property-synonyms, bart-paraphrase, bert-adjectives, bert-domain-synonyms'
    });
    parser.add_argument('--batch-size', {
        required: false,
        type: Number,
        default: 64,
        help: `The batch size for auto paraphrase`
    });
    parser.add_argument('--pruning', {
        required: false,
        type: Number,
        default: 0.5,
        help: `The minimum fraction required for a candidate to be added`
    });
    parser.add_argument('--parameter-datasets', {
        required: true,
        help: `TSV file containing the paths to datasets for strings and entity types.`
    });
    parser.add_argument('--language-model', {
        required: false,
        default: 'bert-large-uncased',
        help: `A string specifying a model name recognizable by the Transformers package (e.g. bert-base-uncased), `
            + `or a path to the directory where the model is saved`
    });
    parser.add_argument('--paraphraser-model', {
        required: false,
        help: `A path to the directory where the bart paraphraser model is saved`
    });
    parser.add_argument('--gpt2-ordering', {
        required: false,
        default: false,
        action: 'store_true',
        help: `Set to True to use gpt2 to decide where to put value`
    });
    parser.add_argument('--filtering', {
        required: false,
        default: false,
        action: 'store_true',
        help: `Enable filtering for phrase extraction.`
    });
    parser.add_argument('--debug', {
        required: false,
        default: false,
        action: 'store_true',
        help: `Enable debugging, which will output intermediate result into files.`
    });
}

export async function execute(args) {
    const classDefs = await loadClassDefs(args.thingpedia);

    if (!args.algorithms) {
        for (let classDef of classDefs)
            args.output.write(classDef.prettyprint() + '\n');
        args.output.end();
    } else {
        const options = args;
        const constants = await parseConstantFile(args.locale, args.constants);
        const functions = args.functions ? args.functions.split(',') : null;

        for (let classDef of classDefs) {
            const generator = new AnnotationGenerator(classDef, constants, functions, args.parameter_datasets, options);
            const annotatedClassDef = await generator.generate();
            args.output.write(annotatedClassDef.prettyprint());
        }
        args.output.end();
    }

    StreamUtils.waitFinish(args.output);
}

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Silei Xu <silei@cs.stanford.edu>
20
21
22	import * as fs from 'fs';	51✔
23	import assert from 'assert';	51✔
24	import util from 'util';	51✔
25	import * as ThingTalk from 'thingtalk';	51✔
26
27	import * as StreamUtils from '../../lib/utils/stream-utils';	51✔
28
29	import { parseConstantFile } from '../lib/constant-file';	51✔
30
31	import AnnotationGenerator from './lib/annotation-generator';	51✔
32
33	async function loadClassDefs(thingpedia) {
34	const library = ThingTalk.Syntax.parse(await util.promisify(fs.readFile)(thingpedia, { encoding: 'utf8' }));	1✔
35	assert(library instanceof ThingTalk.Ast.Library);	1✔
36	return library.classes;	1✔
37	}
38
39	export function initArgparse(subparsers) {	51✔
40	const parser = subparsers.add_parser('auto-annotate', {	51✔
41	add_help: true,
42	description: "Automatically generate annotations including canonicals"
43	});
44	parser.add_argument('--dataset', {	51✔
45	required: true,
46	choices: ['schemaorg', 'sgd', 'multiwoz', 'wikidata', 'custom'],
47	help: 'The dataset to run autoQA on.'
48	});
49	parser.add_argument('-o', '--output', {	51✔
50	required: true,
51	type: fs.createWriteStream
52	});
53	parser.add_argument('-l', '--locale', {	51✔
54	default: 'en-US',
55	help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
56	});
57	parser.add_argument('--constants', {	51✔
58	required: true,
59	help: 'TSV file containing constant values to use.'
60	});
61	parser.add_argument('--thingpedia', {	51✔
62	required: true,
63	help: 'Path to ThingTalk file containing class definitions.'
64	});
65	parser.add_argument('--functions', {	51✔
66	required: false,
67	default: null,
68	help: `List of functions to include, split by comma (no space). Include all functions if not specified`,
69	});
70	parser.add_argument('--remove-existing-canonicals', {	51✔
71	action: 'store_true',
72	help: 'Remove all existing canonical annotations in the schema',
73	default: false
74	});
75	parser.add_argument('--algorithms', {	51✔
76	nargs: '*',
77	help: 'Different algorithms to generate canonicals including bert-property-synonyms, bart-paraphrase, bert-adjectives, bert-domain-synonyms'
78	});
79	parser.add_argument('--batch-size', {	51✔
80	required: false,
81	type: Number,
82	default: 64,
83	help: `The batch size for auto paraphrase`
84	});
85	parser.add_argument('--pruning', {	51✔
86	required: false,
87	type: Number,
88	default: 0.5,
89	help: `The minimum fraction required for a candidate to be added`
90	});
91	parser.add_argument('--parameter-datasets', {	51✔
92	required: true,
93	help: `TSV file containing the paths to datasets for strings and entity types.`
94	});
95	parser.add_argument('--language-model', {	51✔
96	required: false,
97	default: 'bert-large-uncased',
98	help: `A string specifying a model name recognizable by the Transformers package (e.g. bert-base-uncased), `
99	+ `or a path to the directory where the model is saved`
100	});
101	parser.add_argument('--paraphraser-model', {	51✔
102	required: false,
103	help: `A path to the directory where the bart paraphraser model is saved`
104	});
105	parser.add_argument('--gpt2-ordering', {	51✔
106	required: false,
107	default: false,
108	action: 'store_true',
109	help: `Set to True to use gpt2 to decide where to put value`
110	});
111	parser.add_argument('--filtering', {	51✔
112	required: false,
113	default: false,
114	action: 'store_true',
115	help: `Enable filtering for phrase extraction.`
116	});
117	parser.add_argument('--debug', {	51✔
118	required: false,
119	default: false,
120	action: 'store_true',
121	help: `Enable debugging, which will output intermediate result into files.`
122	});
123	}
124
125	export async function execute(args) {	51✔
126	const classDefs = await loadClassDefs(args.thingpedia);	1✔
127
128	if (!args.algorithms) {	1!
UNCOV 129	for (let classDef of classDefs)	×
UNCOV 130	args.output.write(classDef.prettyprint() + '\n');	×
UNCOV 131	args.output.end();	×
132	} else {
133	const options = args;	1✔
134	const constants = await parseConstantFile(args.locale, args.constants);	1✔
135	const functions = args.functions ? args.functions.split(',') : null;	1!
136
137	for (let classDef of classDefs) {	1✔
138	const generator = new AnnotationGenerator(classDef, constants, functions, args.parameter_datasets, options);	1✔
139	const annotatedClassDef = await generator.generate();	1✔
140	args.output.write(annotatedClassDef.prettyprint());	1✔
141	}
142	args.output.end();	1✔
143	}
144
145	StreamUtils.waitFinish(args.output);	1✔
146	}

stanford-oval / genie-toolkit / 1815

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous