1241

pending completion

Build # 1241

Build Type

Pull #247

travis-ci-com

Committed by

web-flow

Commit Message

Merge ceb34bc5b into 64b94b170

Pull Request Pull Request #247: Add script to check how well synthetic set covers evaluation set

Run Details

6636 of 12712 branches covered (52.2%)

Branch coverage included in aggregate %.

7 of 54 new or added lines in 1 file covered. (12.96%)

23 existing lines in 2 files now uncovered.

15330 of 24473 relevant lines covered (62.64%)

2015445.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

10.94

/tool/compute-synthetic-coverage.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>
"use strict";

const fs = require('fs');
const csvparse = require('csv-parse');
const { requoteProgram } = require('../lib/dataset-tools/requoting');

function waitEnd(stream) {
    return new Promise((resolve, reject) => {
        stream.on('end', resolve);
        stream.on('error', reject);
    });
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('compute-synthetic-coverage', {
            addHelp: true,
            description: "Given a evaluation set, compute the percent of programs appeared in synthetic sentence"
        });
        parser.addArgument('--synthetic-set', {
            required: true,
            type: fs.createReadStream,
            help: 'Path to the file containing the synthetic data (in TSV format: id, utterance, thingtalk)'
        });
        parser.addArgument('--evaluation-set', {
            required: true,
            type: fs.createReadStream,
            help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
        });
    },

    async execute(args) {
        let syntheticSize = 0;
        const syntheticPrograms = {}; // counter of unique programs in synthetic set
        args.synthetic_set.setEncoding('utf8');
        const synthetic = args.synthetic_set.pipe(csvparse({ relax: true, delimiter: '\t' }));
        synthetic.on('data', (line) => {
            const requoted = Array.from(requoteProgram(line[2])).join(' ');
            const requotedNoType = requoted.replace(/(param:[a-zA-Z_.]+):\S+/g, '$1');
            syntheticPrograms[requotedNoType] = (syntheticPrograms[requotedNoType] || 0) + 1;
            syntheticSize += 1;
        });
        await waitEnd(synthetic);

        let newCount = 0;
        const newPrograms = new Set();

        let evaluationSize = 0;
        const evaluationPrograms = {}; // counter of unique programs in evaluation set
        args.evaluation_set.setEncoding('utf8');
        const evaluation = args.evaluation_set.pipe(csvparse({ relax: true, delimiter: '\t', relax_column_count: true }));
        evaluation.on('data', (line) => {
            const candidates = line.slice(2);
            let covered = false;
            let requoted, requotedNoType;
            for (let thingtalk of candidates) {
                requoted = Array.from(requoteProgram(thingtalk)).join(' ');
                requotedNoType = requoted.replace(/(param:[a-zA-Z_.]+):\S+/g, '$1');
                if (requotedNoType in syntheticPrograms) {
                    covered = true;
                    break;
                }
            }
            if (!covered) {
                newPrograms.add(requotedNoType);
                newCount += 1;
            }
            evaluationPrograms[requotedNoType] = (evaluationPrograms[requotedNoType] || 0) + 1;
            evaluationSize += 1;
        });
        await waitEnd(evaluation);

        console.log(`${Object.keys(syntheticPrograms).length} unique programs in synthetic set`);
        console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);
        console.log(`${newPrograms.size} programs are not covered.`);
        const coverage = (newCount * 100 / evaluationSize).toFixed(2);
        console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);

        console.log(`% in evaluation set\t% in synthetic set\tprogram`);
        let sumPercentInSynthetic = 0;
        for (let program in evaluationPrograms) {
            const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);
            const percentInSynthetic = ((syntheticPrograms[program] || 0) * 100 / syntheticSize);
            sumPercentInSynthetic += percentInSynthetic;
            console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInSynthetic.toFixed(2)}%\t${program}`);
        }
        console.log(`100.00%\t${sumPercentInSynthetic.toFixed(2)}%`);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Silei Xu <silei@cs.stanford.edu>
20	"use strict";
21
22	const fs = require('fs');	55✔
23	const csvparse = require('csv-parse');	55✔
24	const { requoteProgram } = require('../lib/dataset-tools/requoting');	55✔
25
26	function waitEnd(stream) {
NEW 27	return new Promise((resolve, reject) => {	×
NEW 28	stream.on('end', resolve);	×
NEW 29	stream.on('error', reject);	×
30	});
31	}
32
33	module.exports = {	55✔
34	initArgparse(subparsers) {
35	const parser = subparsers.addParser('compute-synthetic-coverage', {	55✔
36	addHelp: true,
37	description: "Given a evaluation set, compute the percent of programs appeared in synthetic sentence"
38	});
39	parser.addArgument('--synthetic-set', {	55✔
40	required: true,
41	type: fs.createReadStream,
42	help: 'Path to the file containing the synthetic data (in TSV format: id, utterance, thingtalk)'
43	});
44	parser.addArgument('--evaluation-set', {	55✔
45	required: true,
46	type: fs.createReadStream,
47	help: `Path to the file containing the evaluation data (in TSV format: id, utterance, thingtalk)`
48	});
49	},
50
51	async execute(args) {
NEW 52	let syntheticSize = 0;	×
NEW 53	const syntheticPrograms = {}; // counter of unique programs in synthetic set	×
NEW 54	args.synthetic_set.setEncoding('utf8');	×
NEW 55	const synthetic = args.synthetic_set.pipe(csvparse({ relax: true, delimiter: '\t' }));	×
NEW 56	synthetic.on('data', (line) => {	×
NEW 57	const requoted = Array.from(requoteProgram(line[2])).join(' ');	×
NEW 58	const requotedNoType = requoted.replace(/(param:[a-zA-Z_.]+):\S+/g, '$1');	×
NEW 59	syntheticPrograms[requotedNoType] = (syntheticPrograms[requotedNoType] \|\| 0) + 1;	×
NEW 60	syntheticSize += 1;	×
61	});
NEW 62	await waitEnd(synthetic);	×
63
NEW 64	let newCount = 0;	×
NEW 65	const newPrograms = new Set();	×
66
NEW 67	let evaluationSize = 0;	×
NEW 68	const evaluationPrograms = {}; // counter of unique programs in evaluation set	×
NEW 69	args.evaluation_set.setEncoding('utf8');	×
NEW 70	const evaluation = args.evaluation_set.pipe(csvparse({ relax: true, delimiter: '\t', relax_column_count: true }));	×
NEW 71	evaluation.on('data', (line) => {	×
NEW 72	const candidates = line.slice(2);	×
NEW 73	let covered = false;	×
74	let requoted, requotedNoType;
NEW 75	for (let thingtalk of candidates) {	×
NEW 76	requoted = Array.from(requoteProgram(thingtalk)).join(' ');	×
NEW 77	requotedNoType = requoted.replace(/(param:[a-zA-Z_.]+):\S+/g, '$1');	×
NEW 78	if (requotedNoType in syntheticPrograms) {	×
NEW 79	covered = true;	×
NEW 80	break;	×
81	}
82	}
NEW 83	if (!covered) {	×
NEW 84	newPrograms.add(requotedNoType);	×
NEW 85	newCount += 1;	×
86	}
NEW 87	evaluationPrograms[requotedNoType] = (evaluationPrograms[requotedNoType] \|\| 0) + 1;	×
NEW 88	evaluationSize += 1;	×
89	});
NEW 90	await waitEnd(evaluation);	×
91
NEW 92	console.log(`${Object.keys(syntheticPrograms).length} unique programs in synthetic set`);	×
NEW 93	console.log(`${Object.keys(evaluationPrograms).length} unique programs in evaluation set`);	×
NEW 94	console.log(`${newPrograms.size} programs are not covered.`);	×
NEW 95	const coverage = (newCount * 100 / evaluationSize).toFixed(2);	×
NEW 96	console.log(`In total, ${coverage}% (${newCount} / ${evaluationSize}) evaluation examples are not covered.`);	×
97
NEW 98	console.log(`% in evaluation set\t% in synthetic set\tprogram`);	×
NEW 99	let sumPercentInSynthetic = 0;	×
NEW 100	for (let program in evaluationPrograms) {	×
NEW 101	const percentInEvaluation = (evaluationPrograms[program] * 100 / evaluationSize);	×
NEW 102	const percentInSynthetic = ((syntheticPrograms[program] \|\| 0) * 100 / syntheticSize);	×
NEW 103	sumPercentInSynthetic += percentInSynthetic;	×
NEW 104	console.log(`${percentInEvaluation.toFixed(2)}%\t${percentInSynthetic.toFixed(2)}%\t${program}`);	×
105	}
NEW 106	console.log(`100.00%\t${sumPercentInSynthetic.toFixed(2)}%`);	×
107	}
108	};

stanford-oval / genie-toolkit / 1241

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous