1010

pending completion

Build # 1010

Build Type

Pull #212

travis-ci-com

Committed by

web-flow

Commit Message

Merge d34335fd6 into da825f499

Pull Request Pull Request #212: Fixes to evaluate Thingpedia

Run Details

5199 of 12190 branches covered (42.65%)

Branch coverage included in aggregate %.

13 of 90 new or added lines in 6 files covered. (14.44%)

2408 existing lines in 58 files now uncovered.

12841 of 23696 relevant lines covered (54.19%)

2269462.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

23.96

/tool/evaluate-server.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const Tp = require('thingpedia');
const ThingTalk = require('thingtalk');
const fs = require('fs');

const { DatasetParser } = require('../lib/dataset-tools/parsers');
const { SentenceEvaluatorStream, CollectSentenceStatistics } = require('../lib/dataset-tools/evaluation/sentence_evaluator');
const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');
const ParserClient = require('../lib/prediction/parserclient');

function csvDisplay(args, complexity, result, with_numeric=false, without_numeric=false) {
    let buffer = '';
    if (args.csv_prefix)
        buffer = args.csv_prefix + ',';

    let prefix = '';
    if (with_numeric) {
        prefix = `with_numeric_`;
        if (!result[`${prefix}total`])
            return;

        buffer += `with_numeric,` + String(result[`${prefix}total`]);
    } else if (without_numeric) {
        prefix = `without_numeric_`;
        if (!result[`${prefix}total`])
            return;

        buffer += `without_numeric,` + String(result[`${prefix}total`]);
    } else if (complexity === null) {
        buffer += 'all,';
        buffer += String(result.total);
    } else {
        prefix = `complexity_${complexity}/`;
        if (!result[`${prefix}total`])
            return;

        buffer += String(complexity) + ',' + String(result[`${prefix}total`]);
    }
    for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {
        const fullkey = `${prefix}${key}`;
        result[fullkey].length = parseInt(process.env.CSV_LENGTH || 1);
        buffer += ',';
        buffer += String(result[fullkey]);
    }

    args.output.write(buffer + '\n');
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('evaluate-server', {
            addHelp: true,
            description: "Evaluate a trained model on a Genie-generated dataset, by contacting a running Genie server."
        });
        parser.addArgument(['-o', '--output'], {
            required: false,
            defaultValue: process.stdout,
            type: fs.createWriteStream,
            description: "Write results to this file instead of stdout"
        });
        parser.addArgument('--url', {
            required: false,
            help: "URL of the server to evaluate. Use a file:// URL pointing to a model directory to evaluate using a local instance of decanlp",
            defaultValue: 'http://127.0.0.1:8400',
        });
        parser.addArgument('--tokenized', {
            required: false,
            action: 'storeTrue',
            defaultValue: true,
            help: "The dataset is already tokenized (this is the default)."
        });
        parser.addArgument('--no-tokenized', {
            required: false,
            dest: 'tokenized',
            action: 'storeFalse',
            help: "The dataset is not already tokenized."
        });
        parser.addArgument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.addArgument('--contextual', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Process a contextual dataset.',
            defaultValue: false
        });
        parser.addArgument('input_file', {
            nargs: '+',
            type: maybeCreateReadStream,
            help: 'Input datasets to evaluate (in TSV format); use - for standard input'
        });
        parser.addArgument(['-l', '--locale'], {
            required: false,
            defaultValue: 'en-US',
            help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
        });
        parser.addArgument('--debug', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Enable debugging.',
            defaultValue: true
        });
        parser.addArgument('--no-debug', {
            nargs: 0,
            action: 'storeFalse',
            dest: 'debug',
            help: 'Disable debugging.',
        });
        parser.addArgument('--csv', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Output a single CSV line',
        });
        parser.addArgument('--csv-prefix', {
            required: false,
            defaultValue: '',
            help: `Prefix all output lines with this string`
        });
        parser.addArgument('--complexity-metric', {
            choices: ['num_params', 'turn_number'],
            defaultValue: 'num_params',
            help: `Complexity metric to use to divide examples by complexity`
        });
        parser.addArgument('--max-complexity', {
            required: false,
            type: Number,
            defaultValue: '',
            help: 'Collapse all examples of complexity greater or equal to this',
        });
    },

    async execute(args) {
        const tpClient = new Tp.FileClient(args);
        const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);
        const parser = ParserClient.get(args.url, args.locale);
        await parser.start();

        const output = readAllLines(args.input_file)
            .pipe(new DatasetParser({ contextual: args.contextual, preserveId: true, parseMultiplePrograms: true }))
            .pipe(new SentenceEvaluatorStream(parser, schemas, args.tokenized, args.debug, args.complexity_metric))
            .pipe(new CollectSentenceStatistics({ maxComplexity: args.max_complexity }));

        const result = await output.read();

        if (args.csv) {
            csvDisplay(args, null, result);
            if (args.max_complexity) {
                for (let complexity = 0; complexity < args.max_complexity; complexity++)
                    csvDisplay(args, complexity, result);
                csvDisplay(args, '>=' + args.max_complexity, result);
            } else {
                for (let complexity = 0; complexity < 10; complexity++)
                    csvDisplay(args, complexity, result);
            }
            csvDisplay(args, null, result, true);
            csvDisplay(args, null, result, false, true);
        } else {
            for (let key in result) {
                if (Array.isArray(result[key]))
                    args.output.write(`${key} = [${result[key].join(', ')}]\n`);
                else
                    args.output.write(`${key} = ${result[key]}\n`);
            }
        }
        args.output.end();

        await parser.stop();
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const Tp = require('thingpedia');	26✔
23	const ThingTalk = require('thingtalk');	26✔
24	const fs = require('fs');	26✔
25
26	const { DatasetParser } = require('../lib/dataset-tools/parsers');	26✔
27	const { SentenceEvaluatorStream, CollectSentenceStatistics } = require('../lib/dataset-tools/evaluation/sentence_evaluator');	26✔
28	const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');	26✔
29	const ParserClient = require('../lib/prediction/parserclient');	26✔
30
31	function csvDisplay(args, complexity, result, with_numeric=false, without_numeric=false) {	×
UNCOV 32	let buffer = '';	×
UNCOV 33	if (args.csv_prefix)	×
UNCOV 34	buffer = args.csv_prefix + ',';	×
35
UNCOV 36	let prefix = '';	×
UNCOV 37	if (with_numeric) {	×
UNCOV 38	prefix = `with_numeric_`;	×
UNCOV 39	if (!result[`${prefix}total`])	×
UNCOV 40	return;	×
41
42	buffer += `with_numeric,` + String(result[`${prefix}total`]);	×
UNCOV 43	} else if (without_numeric) {	×
UNCOV 44	prefix = `without_numeric_`;	×
UNCOV 45	if (!result[`${prefix}total`])	×
46	return;	×
47
UNCOV 48	buffer += `without_numeric,` + String(result[`${prefix}total`]);	×
UNCOV 49	} else if (complexity === null) {	×
UNCOV 50	buffer += 'all,';	×
UNCOV 51	buffer += String(result.total);	×
52	} else {
UNCOV 53	prefix = `complexity_${complexity}/`;	×
UNCOV 54	if (!result[`${prefix}total`])	×
UNCOV 55	return;	×
56
UNCOV 57	buffer += String(complexity) + ',' + String(result[`${prefix}total`]);	×
58	}
UNCOV 59	for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {	×
UNCOV 60	const fullkey = `${prefix}${key}`;	×
UNCOV 61	result[fullkey].length = parseInt(process.env.CSV_LENGTH \|\| 1);	×
UNCOV 62	buffer += ',';	×
UNCOV 63	buffer += String(result[fullkey]);	×
64	}
65
UNCOV 66	args.output.write(buffer + '\n');	×
67	}
68
69	module.exports = {	26✔
70	initArgparse(subparsers) {
71	const parser = subparsers.addParser('evaluate-server', {	26✔
72	addHelp: true,
73	description: "Evaluate a trained model on a Genie-generated dataset, by contacting a running Genie server."
74	});
75	parser.addArgument(['-o', '--output'], {	26✔
76	required: false,
77	defaultValue: process.stdout,
78	type: fs.createWriteStream,
79	description: "Write results to this file instead of stdout"
80	});
81	parser.addArgument('--url', {	26✔
82	required: false,
83	help: "URL of the server to evaluate. Use a file:// URL pointing to a model directory to evaluate using a local instance of decanlp",
84	defaultValue: 'http://127.0.0.1:8400',
85	});
86	parser.addArgument('--tokenized', {	26✔
87	required: false,
88	action: 'storeTrue',
89	defaultValue: true,
90	help: "The dataset is already tokenized (this is the default)."
91	});
92	parser.addArgument('--no-tokenized', {	26✔
93	required: false,
94	dest: 'tokenized',
95	action: 'storeFalse',
96	help: "The dataset is not already tokenized."
97	});
98	parser.addArgument('--thingpedia', {	26✔
99	required: true,
100	help: 'Path to ThingTalk file containing class definitions.'
101	});
102	parser.addArgument('--contextual', {	26✔
103	nargs: 0,
104	action: 'storeTrue',
105	help: 'Process a contextual dataset.',
106	defaultValue: false
107	});
108	parser.addArgument('input_file', {	26✔
109	nargs: '+',
110	type: maybeCreateReadStream,
111	help: 'Input datasets to evaluate (in TSV format); use - for standard input'
112	});
113	parser.addArgument(['-l', '--locale'], {	26✔
114	required: false,
115	defaultValue: 'en-US',
116	help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
117	});
118	parser.addArgument('--debug', {	26✔
119	nargs: 0,
120	action: 'storeTrue',
121	help: 'Enable debugging.',
122	defaultValue: true
123	});
124	parser.addArgument('--no-debug', {	26✔
125	nargs: 0,
126	action: 'storeFalse',
127	dest: 'debug',
128	help: 'Disable debugging.',
129	});
130	parser.addArgument('--csv', {	26✔
131	nargs: 0,
132	action: 'storeTrue',
133	help: 'Output a single CSV line',
134	});
135	parser.addArgument('--csv-prefix', {	26✔
136	required: false,
137	defaultValue: '',
138	help: `Prefix all output lines with this string`
139	});
140	parser.addArgument('--complexity-metric', {	26✔
141	choices: ['num_params', 'turn_number'],
142	defaultValue: 'num_params',
143	help: `Complexity metric to use to divide examples by complexity`
144	});
145	parser.addArgument('--max-complexity', {	26✔
146	required: false,
147	type: Number,
148	defaultValue: '',
149	help: 'Collapse all examples of complexity greater or equal to this',
150	});
151	},
152
153	async execute(args) {
UNCOV 154	const tpClient = new Tp.FileClient(args);	×
UNCOV 155	const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);	×
UNCOV 156	const parser = ParserClient.get(args.url, args.locale);	×
UNCOV 157	await parser.start();	×
158
UNCOV 159	const output = readAllLines(args.input_file)	×
160	.pipe(new DatasetParser({ contextual: args.contextual, preserveId: true, parseMultiplePrograms: true }))
161	.pipe(new SentenceEvaluatorStream(parser, schemas, args.tokenized, args.debug, args.complexity_metric))
162	.pipe(new CollectSentenceStatistics({ maxComplexity: args.max_complexity }));
163
UNCOV 164	const result = await output.read();	×
165
UNCOV 166	if (args.csv) {	×
UNCOV 167	csvDisplay(args, null, result);	×
UNCOV 168	if (args.max_complexity) {	×
UNCOV 169	for (let complexity = 0; complexity < args.max_complexity; complexity++)	×
UNCOV 170	csvDisplay(args, complexity, result);	×
UNCOV 171	csvDisplay(args, '>=' + args.max_complexity, result);	×
172	} else {
173	for (let complexity = 0; complexity < 10; complexity++)	×
174	csvDisplay(args, complexity, result);	×
175	}
UNCOV 176	csvDisplay(args, null, result, true);	×
UNCOV 177	csvDisplay(args, null, result, false, true);	×
178	} else {
179	for (let key in result) {	×
180	if (Array.isArray(result[key]))	×
181	args.output.write(`${key} = [${result[key].join(', ')}]\n`);	×
182	else
183	args.output.write(`${key} = ${result[key]}\n`);	×
184	}
185	}
UNCOV 186	args.output.end();	×
187
UNCOV 188	await parser.stop();	×
189	}
190	};

stanford-oval / genie-toolkit / 1010

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous