1246

pending completion

Build # 1246

Build Type

Pull #247

travis-ci-com

Committed by

web-flow

Commit Message

Merge e8ab1ff0c into 64b94b170

Pull Request Pull Request #247: Add script to check how well synthetic set covers evaluation set

Run Details

6632 of 12712 branches covered (52.17%)

Branch coverage included in aggregate %.

13 of 63 new or added lines in 1 file covered. (20.63%)

16 existing lines in 3 files now uncovered.

15341 of 24482 relevant lines covered (62.66%)

1999762.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.81

/tool/compute-training-coverage.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>
"use strict";

const fs = require('fs');
const ThingTalk = require('thingtalk');
const Tp = require('thingpedia');

const Utils = require('../lib/utils/misc-utils');
const { requoteProgram } = require('../lib/dataset-tools/requoting');
const { readAllLines } = require('./lib/argutils');
const { DatasetParser } = require('../lib/dataset-tools/parsers');

async function normalize(preprocessed, target_code, schemas) {
    const entities = Utils.makeDummyEntities(preprocessed);
    const sequence = target_code.split(' ');
    const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);
    await parsed.typecheck(schemas);
    const normalized = ThingTalk.NNSyntax.toNN(parsed, preprocessed, {}, {
        allocateEntities: true,
        typeAnnotations: false
    });
    return normalized.join(' ');
}


module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('compute-training-coverage', {
            addHelp: true,
            description: "Given a evaluation set, compute the percent of programs appeared in training set"
        });
        parser.add_argument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.add_argument('-l', '--locale', {
            required: false,
            default: 'en-US',
            help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
        });
        parser.addArgument('--training-set', {
            required: true,
            type: fs.createReadStream,
            help: 'Path to the file containing the training data (in TSV format: id, utterance, thingtalk)'
        });
        parser.addArgument('--evaluation-set', {
            required: true,
            type: fs.createReadStream,
            help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
        });
    },

    async execute(args) {
        const tpClient = new Tp.FileClient(args);
        const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);

        let trainingSize = 0;
        const trainingPrograms = new Map(); // counter of unique programs in training set
        args.training_set.setEncoding('utf8');
        const training = await readAllLines([args.training_set])
            .pipe(new DatasetParser({ preserveId: true }));
        for await (const line of training) {
            const normalized = await normalize(line.preprocessed, line.target_code, schemas);
            const requoted = Array.from(requoteProgram(normalized)).join(' ');
            trainingPrograms[requoted] = (trainingPrograms[requoted] || 0) + 1;
            trainingSize += 1;
        }

        let newCount = 0;
        const newPrograms = new Set();

        let evaluationSize = 0;
        const evaluationPrograms = new Map(); // counter of unique programs in evaluation set
        args.evaluation_set.setEncoding('utf8');
        const evaluation = await readAllLines([args.evaluation_set])
            .pipe(new DatasetParser({ preserveId: true, parseMultiplePrograms: true }));
        for await (const line of evaluation) {
            const candidates = line.target_code;
            let covered = false;
            let requoted;
            for (let thingtalk of candidates) {
                const normalized = await normalize(line.preprocessed, thingtalk, schemas);
                requoted = Array.from(requoteProgram(normalized)).join(' ');
                if (requoted in trainingPrograms) {
                    covered = true;
                    break;
                }
            }
            if (!covered) {
                newPrograms.add(requoted);
                newCount += 1;
            }
            evaluationPrograms[requoted] = (evaluationPrograms[requoted] || 0) + 1;
            evaluationSize += 1;
        }

        console.log(`${Object.keys(trainingPrograms).length} unique programs in training set`);
        console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);
        console.log(`${newPrograms.size} programs are not covered.`);
        const coverage = (newCount * 100 / evaluationSize).toFixed(2);
        console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);

        console.log(`% in evaluation set\t% in training set\tprogram`);
        let sumPercentInTraining = 0;
        for (let program in evaluationPrograms) {
            const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);
            const percentInTraining = ((trainingPrograms[program] || 0) * 100 / trainingSize);
            sumPercentInTraining += percentInTraining;
            console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInTraining.toFixed(2)}%\t${program}`);
        }
        console.log(`100.00%\t${sumPercentInTraining.toFixed(2)}%`);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Silei Xu <silei@cs.stanford.edu>
20	"use strict";
21
22	const fs = require('fs');	55✔
23	const ThingTalk = require('thingtalk');	55✔
24	const Tp = require('thingpedia');	55✔
25
26	const Utils = require('../lib/utils/misc-utils');	55✔
27	const { requoteProgram } = require('../lib/dataset-tools/requoting');	55✔
28	const { readAllLines } = require('./lib/argutils');	55✔
29	const { DatasetParser } = require('../lib/dataset-tools/parsers');	55✔
30
31	async function normalize(preprocessed, target_code, schemas) {
NEW 32	const entities = Utils.makeDummyEntities(preprocessed);	×
NEW 33	const sequence = target_code.split(' ');	×
NEW 34	const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);	×
NEW 35	await parsed.typecheck(schemas);	×
NEW 36	const normalized = ThingTalk.NNSyntax.toNN(parsed, preprocessed, {}, {	×
37	allocateEntities: true,
38	typeAnnotations: false
39	});
NEW 40	return normalized.join(' ');	×
41	}
42
43
44	module.exports = {	55✔
45	initArgparse(subparsers) {
46	const parser = subparsers.addParser('compute-training-coverage', {	55✔
47	addHelp: true,
48	description: "Given a evaluation set, compute the percent of programs appeared in training set"
49	});
50	parser.add_argument('--thingpedia', {	55✔
51	required: true,
52	help: 'Path to ThingTalk file containing class definitions.'
53	});
54	parser.add_argument('-l', '--locale', {	55✔
55	required: false,
56	default: 'en-US',
57	help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
58	});
59	parser.addArgument('--training-set', {	55✔
60	required: true,
61	type: fs.createReadStream,
62	help: 'Path to the file containing the training data (in TSV format: id, utterance, thingtalk)'
63	});
64	parser.addArgument('--evaluation-set', {	55✔
65	required: true,
66	type: fs.createReadStream,
67	help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
68	});
69	},
70
71	async execute(args) {
NEW 72	const tpClient = new Tp.FileClient(args);	×
NEW 73	const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);	×
74
NEW 75	let trainingSize = 0;	×
NEW 76	const trainingPrograms = new Map(); // counter of unique programs in training set	×
NEW 77	args.training_set.setEncoding('utf8');	×
NEW 78	const training = await readAllLines([args.training_set])	×
79	.pipe(new DatasetParser({ preserveId: true }));
NEW 80	for await (const line of training) {	×
NEW 81	const normalized = await normalize(line.preprocessed, line.target_code, schemas);	×
NEW 82	const requoted = Array.from(requoteProgram(normalized)).join(' ');	×
NEW 83	trainingPrograms[requoted] = (trainingPrograms[requoted] \|\| 0) + 1;	×
NEW 84	trainingSize += 1;	×
85	}
86
NEW 87	let newCount = 0;	×
NEW 88	const newPrograms = new Set();	×
89
NEW 90	let evaluationSize = 0;	×
NEW 91	const evaluationPrograms = new Map(); // counter of unique programs in evaluation set	×
NEW 92	args.evaluation_set.setEncoding('utf8');	×
NEW 93	const evaluation = await readAllLines([args.evaluation_set])	×
94	.pipe(new DatasetParser({ preserveId: true, parseMultiplePrograms: true }));
NEW 95	for await (const line of evaluation) {	×
NEW 96	const candidates = line.target_code;	×
NEW 97	let covered = false;	×
98	let requoted;
NEW 99	for (let thingtalk of candidates) {	×
NEW 100	const normalized = await normalize(line.preprocessed, thingtalk, schemas);	×
NEW 101	requoted = Array.from(requoteProgram(normalized)).join(' ');	×
NEW 102	if (requoted in trainingPrograms) {	×
NEW 103	covered = true;	×
NEW 104	break;	×
105	}
106	}
NEW 107	if (!covered) {	×
NEW 108	newPrograms.add(requoted);	×
NEW 109	newCount += 1;	×
110	}
NEW 111	evaluationPrograms[requoted] = (evaluationPrograms[requoted] \|\| 0) + 1;	×
NEW 112	evaluationSize += 1;	×
113	}
114
NEW 115	console.log(`${Object.keys(trainingPrograms).length} unique programs in training set`);	×
NEW 116	console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);	×
NEW 117	console.log(`${newPrograms.size} programs are not covered.`);	×
NEW 118	const coverage = (newCount * 100 / evaluationSize).toFixed(2);	×
NEW 119	console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);	×
120
NEW 121	console.log(`% in evaluation set\t% in training set\tprogram`);	×
NEW 122	let sumPercentInTraining = 0;	×
NEW 123	for (let program in evaluationPrograms) {	×
NEW 124	const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);	×
NEW 125	const percentInTraining = ((trainingPrograms[program] \|\| 0) * 100 / trainingSize);	×
NEW 126	sumPercentInTraining += percentInTraining;	×
NEW 127	console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInTraining.toFixed(2)}%\t${program}`);	×
128	}
NEW 129	console.log(`100.00%\t${sumPercentInTraining.toFixed(2)}%`);	×
130	}
131	};

stanford-oval / genie-toolkit / 1246

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous