1909

pending completion

Build # 1909

Build Type

Pull #564

travis-ci-com

Committed by

web-flow

Commit Message

Merge e98409694 into fb8b240ba

Pull Request Pull Request #564: Changes to enable Genie for COVID

Run Details

12035 of 25556 branches covered (47.09%)

Branch coverage included in aggregate %.

7 of 10 new or added lines in 5 files covered. (70.0%)

31 existing lines in 5 files now uncovered.

28215 of 48232 relevant lines covered (58.5%)

378703.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33

/tool/retokenize-evaluation-data.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as argparse from 'argparse';
import * as fs from 'fs';
import Stream from 'stream';
import csvparse from 'csv-parse';

import EnglishTokenizer from '../lib/i18n/tokenizer/english';
import { DatasetStringifier, StreamUtils } from '../lib';

const rawUtterances : string[] = [];
class Validator extends Stream.Transform {
    private _update_id;
    private _id_prefix;
    private _counter;
    private _tokenizer;

    constructor(options : { update_id : boolean, id_prefix ?: string }) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._tokenizer = new EnglishTokenizer();
        this._update_id = options.update_id;
        this._id_prefix = options.id_prefix || '';
        this._counter = 0;
    }

    _transform(row : Record<string, string>, encoding : BufferEncoding, callback : () => void) {
        const id = this._update_id ? this._newID() : row.id;
        const raw = rawUtterances[this._counter];
        const tokenized = this._tokenizer.tokenize(raw).tokens;
        this.push({ id, raw, preprocessed: tokenized.join(' '), target_code: row.thingtalk });
        this._counter += 1;
        callback();
    }

    _newID() : string {
        let id = String(this._counter + 1);
        while (id.length < 3) 
            id = '0' + id;
        return this._id_prefix + id;
    }

    _flush(callback : () => void) {
        process.nextTick(callback);
    }
}

class RawUttearnceLoader extends Stream.Transform {
    constructor() {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });
    }

    _transform(row : Record<string, string>, encoding : BufferEncoding, callback : () => void) {
        rawUtterances.push(row.raw.normalize('NFD').replace(/[\u0300-\u036f]/g, ''));
        callback();
    }

    _flush(callback : () => void) {
        process.nextTick(callback);
    }
}

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('retokenize-eval', {
        add_help: true,
        description: "Retokenize the evaluation dataset."
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream,
        help: 'Path to the updated evaluation dataset'
    });
    parser.add_argument('input', {
        type: fs.createReadStream,
        help: 'Path to the original evaluation dataset'
    });
    parser.add_argument('--raw', {
        type: fs.createReadStream,
        help: 'Path to the file with raw utterances for the evaluation dataset'
    });
    parser.add_argument('--update-id', {
        action: 'store_true',
        default: false,
        help: 'reorder ids for the examples starting from 1'
    });
    parser.add_argument('--id-prefix', {
        default: '',
        required: false,
        help: 'prefix of new id'
    });
}

export async function execute(args : any) {
    const loader = new RawUttearnceLoader();
    await args.raw
        .pipe(csvparse({ columns: ['id', 'raw'], delimiter: '\t', relax: true }))
        .pipe(loader);

    await StreamUtils.waitFinish(loader);

    await args.input
        .pipe(csvparse({ columns: ['id', 'tokenized', 'thingtalk'], delimiter: '\t', relax: true }))
        .pipe(new Validator(args))
        .pipe(new DatasetStringifier())
        .pipe(args.output);
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as argparse from 'argparse';
22	import * as fs from 'fs';	90✔
23	import Stream from 'stream';	90✔
24	import csvparse from 'csv-parse';	90✔
25
26	import EnglishTokenizer from '../lib/i18n/tokenizer/english';	90✔
27	import { DatasetStringifier, StreamUtils } from '../lib';	90✔
28
29	const rawUtterances : string[] = [];	90✔
30	class Validator extends Stream.Transform {
31	private _update_id;
32	private _id_prefix;
33	private _counter;
34	private _tokenizer;
35
36	constructor(options : { update_id : boolean, id_prefix ?: string }) {
UNCOV 37	super({	×
38	readableObjectMode: true,
39	writableObjectMode: true
40	});
41
42	this._tokenizer = new EnglishTokenizer();	×
43	this._update_id = options.update_id;	×
44	this._id_prefix = options.id_prefix \|\| '';	×
UNCOV 45	this._counter = 0;	×
46	}
47
48	_transform(row : Record<string, string>, encoding : BufferEncoding, callback : () => void) {
UNCOV 49	const id = this._update_id ? this._newID() : row.id;	×
UNCOV 50	const raw = rawUtterances[this._counter];	×
UNCOV 51	const tokenized = this._tokenizer.tokenize(raw).tokens;	×
UNCOV 52	this.push({ id, raw, preprocessed: tokenized.join(' '), target_code: row.thingtalk });	×
UNCOV 53	this._counter += 1;	×
UNCOV 54	callback();	×
55	}
56
57	_newID() : string {
UNCOV 58	let id = String(this._counter + 1);	×
UNCOV 59	while (id.length < 3)	×
UNCOV 60	id = '0' + id;	×
UNCOV 61	return this._id_prefix + id;	×
62	}
63
64	_flush(callback : () => void) {
UNCOV 65	process.nextTick(callback);	×
66	}
67	}
68
69	class RawUttearnceLoader extends Stream.Transform {
70	constructor() {
UNCOV 71	super({	×
72	readableObjectMode: true,
73	writableObjectMode: true
74	});
75	}
76
77	_transform(row : Record<string, string>, encoding : BufferEncoding, callback : () => void) {
UNCOV 78	rawUtterances.push(row.raw.normalize('NFD').replace(/[\u0300-\u036f]/g, ''));	×
UNCOV 79	callback();	×
80	}
81
82	_flush(callback : () => void) {
UNCOV 83	process.nextTick(callback);	×
84	}
85	}
86
87	export function initArgparse(subparsers : argparse.SubParser) {	90✔
88	const parser = subparsers.add_parser('retokenize-eval', {	90✔
89	add_help: true,
90	description: "Retokenize the evaluation dataset."
91	});
92	parser.add_argument('-o', '--output', {	90✔
93	required: true,
94	type: fs.createWriteStream,
95	help: 'Path to the updated evaluation dataset'
96	});
97	parser.add_argument('input', {	90✔
98	type: fs.createReadStream,
99	help: 'Path to the original evaluation dataset'
100	});
101	parser.add_argument('--raw', {	90✔
102	type: fs.createReadStream,
103	help: 'Path to the file with raw utterances for the evaluation dataset'
104	});
105	parser.add_argument('--update-id', {	90✔
106	action: 'store_true',
107	default: false,
108	help: 'reorder ids for the examples starting from 1'
109	});
110	parser.add_argument('--id-prefix', {	90✔
111	default: '',
112	required: false,
113	help: 'prefix of new id'
114	});
115	}
116
117	export async function execute(args : any) {	90✔
UNCOV 118	const loader = new RawUttearnceLoader();	×
UNCOV 119	await args.raw	×
120	.pipe(csvparse({ columns: ['id', 'raw'], delimiter: '\t', relax: true }))
121	.pipe(loader);
122
UNCOV 123	await StreamUtils.waitFinish(loader);	×
124
UNCOV 125	await args.input	×
126	.pipe(csvparse({ columns: ['id', 'tokenized', 'thingtalk'], delimiter: '\t', relax: true }))
127	.pipe(new Validator(args))
128	.pipe(new DatasetStringifier())
129	.pipe(args.output);
130	}

stanford-oval / genie-toolkit / 1909

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous