1076

pending completion

Build # 1076

Build Type

Pull #247

travis-ci-com

Committed by

web-flow

Commit Message

Merge 5579f32a6 into f1ddda6ad

Pull Request Pull Request #247: Add script to check how well synthetic set covers evaluation set

Run Details

6759 of 12287 branches covered (55.01%)

Branch coverage included in aggregate %.

7 of 52 new or added lines in 1 file covered. (13.46%)

4 existing lines in 1 file now uncovered.

15416 of 23875 relevant lines covered (64.57%)

2818898.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

11.29

/tool/compute-synthetic-coverage.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>
"use strict";

const fs = require('fs');
const csvparse = require('csv-parse');
const { requoteProgram } = require('../lib/dataset-tools/requoting');

function waitEnd(stream) {
    return new Promise((resolve, reject) => {
        stream.on('end', resolve);
        stream.on('error', reject);
    });
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('compute-synthetic-coverage', {
            addHelp: true,
            description: "Given a evaluation set, compute the percent of programs appeared in synthetic sentence"
        });
        parser.addArgument('--synthetic-set', {
            required: true,
            type: fs.createReadStream,
            help: 'Path to the file containing the synthetic data (in TSV format: id, utterance, thingtalk)'
        });
        parser.addArgument('--evaluation-set', {
            required: true,
            type: fs.createReadStream,
            help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
        });
    },

    async execute(args) {
        let syntheticSize = 0;
        const syntheticPrograms = {}; // counter of unique programs in synthetic set
        args.synthetic_set.setEncoding('utf8');
        const synthetic = args.synthetic_set.pipe(csvparse({ relax: true, delimiter: '\t' }));
        synthetic.on('data', (line) => {
            const requoted = Array.from(requoteProgram(line[2])).join(' ');
            syntheticPrograms[requoted] = (syntheticPrograms[requoted] || 0) + 1;
            syntheticSize += 1;
        });
        await waitEnd(synthetic);

        let newCount = 0;
        const newPrograms = new Set();

        let evaluationSize = 0;
        const evaluationPrograms = {}; // counter of unique programs in evaluation set
        args.evaluation_set.setEncoding('utf8');
        const evaluation = args.evaluation_set.pipe(csvparse({ relax: true, delimiter: '\t', relax_column_count: true }));
        evaluation.on('data', (line) => {
            const candidates = line.slice(2);
            let covered = false;
            let requoted;
            for (let thingtalk of candidates) {
                requoted = Array.from(requoteProgram(thingtalk)).join(' ');
                if (requoted in syntheticPrograms) {
                    covered = true;
                    break;
                }
            }
            if (!covered) {
                newPrograms.add(requoted);
                newCount += 1;
            }
            evaluationPrograms[requoted] = (evaluationPrograms[requoted] || 0) + 1;
            evaluationSize += 1;
        });
        await waitEnd(evaluation);

        console.log(`${Object.keys(syntheticPrograms).length} unique programs in synthetic set`);
        console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);
        console.log(`${newPrograms.size} programs are not covered.`);
        const coverage = (newCount * 100 / evaluationSize).toFixed(2);
        console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);

        console.log(`% in evaluation set\t% in synthetic set\tprogram`);
        let sumPercentInSynthetic = 0;
        for (let program in evaluationPrograms) {
            const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);
            const percentInSynthetic = ((syntheticPrograms[program] || 0) * 100 / syntheticSize);
            sumPercentInSynthetic += percentInSynthetic;
            console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInSynthetic.toFixed(2)}%\t${program}`);
        }
        console.log(`100.00%\t${sumPercentInSynthetic.toFixed(2)}%`);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Silei Xu <silei@cs.stanford.edu>
20	"use strict";
21
22	const fs = require('fs');	54✔
23	const csvparse = require('csv-parse');	54✔
24	const { requoteProgram } = require('../lib/dataset-tools/requoting');	54✔
25
26	function waitEnd(stream) {
NEW 27	return new Promise((resolve, reject) => {	×
NEW 28	stream.on('end', resolve);	×
NEW 29	stream.on('error', reject);	×
30	});
31	}
32
33	module.exports = {	54✔
34	initArgparse(subparsers) {
35	const parser = subparsers.addParser('compute-synthetic-coverage', {	54✔
36	addHelp: true,
37	description: "Given a evaluation set, compute the percent of programs appeared in synthetic sentence"
38	});
39	parser.addArgument('--synthetic-set', {	54✔
40	required: true,
41	type: fs.createReadStream,
42	help: 'Path to the file containing the synthetic data (in TSV format: id, utterance, thingtalk)'
43	});
44	parser.addArgument('--evaluation-set', {	54✔
45	required: true,
46	type: fs.createReadStream,
47	help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
48	});
49	},
50
51	async execute(args) {
NEW 52	let syntheticSize = 0;	×
NEW 53	const syntheticPrograms = {}; // counter of unique programs in synthetic set	×
NEW 54	args.synthetic_set.setEncoding('utf8');	×
NEW 55	const synthetic = args.synthetic_set.pipe(csvparse({ relax: true, delimiter: '\t' }));	×
NEW 56	synthetic.on('data', (line) => {	×
NEW 57	const requoted = Array.from(requoteProgram(line[2])).join(' ');	×
NEW 58	syntheticPrograms[requoted] = (syntheticPrograms[requoted] \|\| 0) + 1;	×
NEW 59	syntheticSize += 1;	×
60	});
NEW 61	await waitEnd(synthetic);	×
62
NEW 63	let newCount = 0;	×
NEW 64	const newPrograms = new Set();	×
65
NEW 66	let evaluationSize = 0;	×
NEW 67	const evaluationPrograms = {}; // counter of unique programs in evaluation set	×
NEW 68	args.evaluation_set.setEncoding('utf8');	×
NEW 69	const evaluation = args.evaluation_set.pipe(csvparse({ relax: true, delimiter: '\t', relax_column_count: true }));	×
NEW 70	evaluation.on('data', (line) => {	×
NEW 71	const candidates = line.slice(2);	×
NEW 72	let covered = false;	×
73	let requoted;
NEW 74	for (let thingtalk of candidates) {	×
NEW 75	requoted = Array.from(requoteProgram(thingtalk)).join(' ');	×
NEW 76	if (requoted in syntheticPrograms) {	×
NEW 77	covered = true;	×
NEW 78	break;	×
79	}
80	}
NEW 81	if (!covered) {	×
NEW 82	newPrograms.add(requoted);	×
NEW 83	newCount += 1;	×
84	}
NEW 85	evaluationPrograms[requoted] = (evaluationPrograms[requoted] \|\| 0) + 1;	×
NEW 86	evaluationSize += 1;	×
87	});
NEW 88	await waitEnd(evaluation);	×
89
NEW 90	console.log(`${Object.keys(syntheticPrograms).length} unique programs in synthetic set`);	×
NEW 91	console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);	×
NEW 92	console.log(`${newPrograms.size} programs are not covered.`);	×
NEW 93	const coverage = (newCount * 100 / evaluationSize).toFixed(2);	×
NEW 94	console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);	×
95
NEW 96	console.log(`% in evaluation set\t% in synthetic set\tprogram`);	×
NEW 97	let sumPercentInSynthetic = 0;	×
NEW 98	for (let program in evaluationPrograms) {	×
NEW 99	const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);	×
NEW 100	const percentInSynthetic = ((syntheticPrograms[program] \|\| 0) * 100 / syntheticSize);	×
NEW 101	sumPercentInSynthetic += percentInSynthetic;	×
NEW 102	console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInSynthetic.toFixed(2)}%\t${program}`);	×
103	}
NEW 104	console.log(`100.00%\t${sumPercentInSynthetic.toFixed(2)}%`);	×
105	}
106	};

stanford-oval / genie-toolkit / 1076

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous