stanford-oval / genie-toolkit / 710

pending completion

Build # 710

Build Type

Pull #152

travis-ci-com

Committed by

web-flow

Commit Message

minor

Pull Request Pull Request #152: Wip/multilanguage

Run Details

1281 of 4728 branches covered (27.09%)

Branch coverage included in aggregate %.

49 of 90 new or added lines in 7 files covered. (54.44%)

3 existing lines in 2 files now uncovered.

3996 of 9577 relevant lines covered (41.72%)

16005.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.62

/tool/requote.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>,
//          Mehrad Moradshahi <mehrad@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const fs = require('fs');
const Stream = require('stream');
const assert = require('assert');

const { DatasetParser, DatasetStringifier } = require('../lib/dataset-parsers');
const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');
const StreamUtils = require('../lib/stream-utils');

const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;


function check_range(index, spansBySentencePos){
    for (let i = 0; i < spansBySentencePos.length; i++){
        const span = spansBySentencePos[i];
        if (index >= span.begin && index < span.end)
            return false;
    }
    return true;

}

function findSubstring(sequence, substring, spansBySentencePos) {
    for (let i = 0; i < sequence.length - substring.length + 1; i++) {
        let found = true;

        for (let j = 0; j < substring.length; j++) {
            if (sequence[i+j] !== substring[j]) {
                found = false;
                break;
            }
        }
        if (found && check_range(i, spansBySentencePos))
            return i;
    }
    return -1;
}


function findSpanType(program, begin_index, end_index) {
    let spanType;
    if (begin_index > 1 && program[begin_index-2] === 'location:') {
        spanType = 'LOCATION';
    } else if (program[begin_index-2] === '==' && program[begin_index-3].endsWith('Number')){
        spanType = 'NUMBER';
    } else if (program[begin_index-2] === '==' && program[begin_index-3].includes('phone_number')){
        spanType = 'PHONE_NUMBER';
    } else if (end_index === program.length - 1 || !program[end_index+1].startsWith('^^')) {
        spanType = 'QUOTED_STRING';
    } else {
        switch (program[end_index+1]) {
        case '^^tt:hashtag':
            spanType = 'HASHTAG';
            break;
        case '^^tt:username':
            spanType = 'USERNAME';
            break;
        default:
            spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);
        }
        end_index++;
    }
    return [spanType, end_index];
}


function createProgram(program, spansByProgramPos, entityRemap) {
    let in_string = false;
    let newProgram = [];
    let programSpanIndex = 0;
    for (let i = 0; i < program.length; i++) {
        let token = program[i];
        if (token === '"') {
            in_string = !in_string;
            if (in_string)
                continue;
            const currentSpan = spansByProgramPos[programSpanIndex];
            try{
                assert(currentSpan.mapTo);
            }
            catch (e) {
                console.log('error!');
                console.log(program.join(' '));
            }

            newProgram.push(currentSpan.mapTo);
            programSpanIndex++;
            continue;
        }
        if (in_string) continue;

        if (token === 'location:' || token.startsWith('^^')) {
            continue;
        } else if (ENTITY_MATCH_REGEX.test(token)) {
            assert(entityRemap[token]);
            newProgram.push(entityRemap[token]);
        } else {
            newProgram.push(token);
        }
    }
    return newProgram;
}

function qpisSentence(sentence, spansBySentencePos) {

    let current_span_idx = 0;
    let current_span = spansBySentencePos[0];

    let newSentence = [];
    let i = 0;
    let in_string = false;
    while (i < sentence.length) {
        let word = sentence[i];
        if (current_span === null || i < current_span.begin) {
            newSentence.push(word);
            i += 1;
        } else if (i >= current_span.end) {
            newSentence.push('"');
            in_string = false;
            current_span_idx += 1;
            current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;
        } else if (i === current_span.begin || in_string) {
            if (i === current_span.begin)
                newSentence.push('"');
            newSentence.push(word);
            in_string = true;
            i += 1;

        } else {
            i += 1;
        }
    }
    if (in_string)
        newSentence.push('"');
    return newSentence;

}

function createSentence(sentence, spansBySentencePos) {

    let current_span_idx = 0;
    let current_span = spansBySentencePos[0];

    let newSentence = [];
    let entityNumbers = {};
    function getEntityNumber(entityType) {
        let nextId = (entityNumbers[entityType] || 0);
        entityNumbers[entityType] = nextId + 1;
        return String(nextId);
    }
    let entityRemap = {};

    let i = 0;
    while (i < sentence.length) {
        let word = sentence[i];
        if (current_span === null || i < current_span.begin) {
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null) {
                // input sentence contains entities
                const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);
                entityRemap[word] = newEntity;
                newSentence.push(newEntity);
            } else {
                newSentence.push(word);
            }
            i += 1;
        } else if (i === current_span.begin) {
            const newEntity = current_span.type + '_' + getEntityNumber(current_span.type);
            current_span.mapTo = newEntity;
            newSentence.push(newEntity);
            i += 1;
        } else if (i >= current_span.end) {
            current_span_idx += 1;
            current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;
        } else {
            i += 1;
        }
    }

    return [newSentence, entityRemap];

}

function sortWithIndeces(toSort, sort_func) {
    let toSort_new = [];
    for (let i = 0; i < toSort.length; i++) 
        toSort_new[i] = [toSort[i], i];
    
    toSort_new.sort(sort_func);
    let sortIndices = [];
    for (let j = 0; j < toSort_new.length; j++) {
        sortIndices.push(toSort_new[j][1]);
        toSort[j] = toSort_new[j][0];
    }
    return sortIndices;
}

function getProgSpans(program, is_quoted) {
    let in_string = false;
    let begin_index = null;
    let end_index = null;
    let all_prog_spans = [];
    if (is_quoted) {
        for (let i = 0; i < program.length; i++) {
            let token = program[i];
            if (ENTITY_MATCH_REGEX.test(token)){
                begin_index = i;
                end_index = begin_index + 1;
                let prog_span = {begin: begin_index, end: end_index};
                all_prog_spans.push(prog_span);
            }
        }

    } else {
        for (let i = 0; i < program.length; i++) {
            let token = program[i];
            if (token === '"') {
                in_string = !in_string;
                if (in_string) {
                    begin_index = i + 1;
                } else {
                    end_index = i;
                    let prog_span = {begin: begin_index, end: end_index};
                    all_prog_spans.push(prog_span);
                }
            }
        }
    }



    // sort params based on length so that longer phrases get matched sooner
    let sort_func = function (a, b)  {
        const {begin:abegin, end:aend} = a[0];
        const {begin:bbegin, end:bend} = b[0];
        return (bend - bbegin) - (aend - abegin);
    };

    // sort array in-place and return sorted indices
    let sortIndices = sortWithIndeces(all_prog_spans, sort_func);

    return [all_prog_spans, sortIndices];
}


function findSpanPositions(id, sentence, program, is_quoted) {
    const spansBySentencePos = [];
    const spansByProgramPos = [];

    const [all_prog_spans_sorted, sortIndices]  = getProgSpans(program, is_quoted);

    for (let i = 0; i < all_prog_spans_sorted.length; i++) {
        const prog_span = all_prog_spans_sorted[i];
        let [begin_index, end_index] = [prog_span.begin, prog_span.end];
        const substring = program.slice(begin_index, end_index);
        const idx = findSubstring(sentence, substring, spansBySentencePos);
        if (idx < 0){
            console.log(program.join(' '));
            console.error('***Error: Program contains some parameters that are not present in the sentence***');
            throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);

        }

        const spanBegin = idx;
        const spanEnd = idx + end_index - begin_index;

        let spanType = findSpanType(program, begin_index, end_index)[0];

        const span = {begin: spanBegin, end: spanEnd, type: spanType, mapTo: undefined};
        spansBySentencePos.push(span);
        spansByProgramPos.push(span);

    }

    return [spansBySentencePos, spansByProgramPos, sortIndices];
}

function requoteSentence(id, sentence, program, mode, is_quoted) {
    sentence = sentence.split(' ');
    program = program.split(' ');

    let [spansBySentencePos, spansByProgramPosSorted, sortIndices] = findSpanPositions(id, sentence, program, is_quoted);

    if (spansBySentencePos.length === 0)
        return [sentence.join(' '), program.join(' ')];

    // revert back the order after matching is done
    let spansByProgramPos = [];
    for (let i = 0; i < sortIndices.length; i++)
        spansByProgramPos[sortIndices[i]] = spansByProgramPosSorted[i];

    spansBySentencePos.sort((a, b) => {
        const {begin:abegin, end:aend} = a;
        const {begin:bbegin, end:bend} = b;
        if (abegin < bbegin)
            return -1;
        if (bbegin < abegin)
            return +1;
        if (aend < bend)
            return -1;
        if (bend < aend)
            return +1;
        return 0;
    });

    let newSentence, newProgram, entityRemap;

    if (mode === 'replace'){
        [newSentence, entityRemap] = createSentence(sentence, spansBySentencePos);
        newProgram = createProgram(program, spansByProgramPos, entityRemap);
    } else if (mode === 'qpis') {
        newSentence = qpisSentence(sentence, spansBySentencePos);
        newProgram = program;
    }

    return [newSentence.join(' '), newProgram.join(' ')];
}


module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('requote', {
            addHelp: true,
            description: "Requote a dataset."
        });
        parser.addArgument(['-o', '--output'], {
            required: true,
            type: fs.createWriteStream
        });
        parser.addArgument('--contextual', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Process a contextual dataset.',
            defaultValue: false
        });
        parser.addArgument('--mode', {
            type: String,
            help: 'Type of requoting (replace with placeholder, or just add quotation marks around params)',
            choices: ['replace', 'qpis'],
            defaultValue: 'replace'
        });
        parser.addArgument('--is_quoted', {
            action: 'storeTrue',
            help: 'The input dataset is already quoted. Pass this to qpis a quoted dataset',
            defaultValue: false
        });
        parser.addArgument('input_file', {
            nargs: '+',
            type: maybeCreateReadStream,
            help: 'Input datasets to evaluate (in TSV format); use - for standard input'
        });
    },

    async execute(args) {
        readAllLines(args.input_file)
            .pipe(new DatasetParser({ contextual: args.contextual }))
            .pipe(new Stream.Transform({
                objectMode: true,

                transform(ex, encoding, callback) {
                    try {
                        const [newSentence, newProgram] = requoteSentence(ex.id, ex.preprocessed, ex.target_code, args.mode, args.is_quoted);
                        ex.preprocessed = newSentence;
                        ex.target_code = newProgram;
                        callback(null, ex);
                    } catch(e) {
                        console.error(`Failed to requote`);
                        console.error(ex.preprocessed);
                        console.error(ex.target_code);
                        callback(e);
                    }
                },

                flush(callback) {
                    process.nextTick(callback);
                }
            }))
            .pipe(new DatasetStringifier())
            .pipe(args.output);

        await StreamUtils.waitFinish(args.output);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>,
8	// Mehrad Moradshahi <mehrad@cs.stanford.edu>
9	//
10	// See COPYING for details
11	"use strict";
12
13	const fs = require('fs');	46✔
14	const Stream = require('stream');	46✔
15	const assert = require('assert');	46✔
16
17	const { DatasetParser, DatasetStringifier } = require('../lib/dataset-parsers');	46✔
18	const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');	46✔
19	const StreamUtils = require('../lib/stream-utils');	46✔
20
21	const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;	46✔
22
23
24	function check_range(index, spansBySentencePos){
25	for (let i = 0; i < spansBySentencePos.length; i++){	104✔
26	const span = spansBySentencePos[i];	80✔
27	if (index >= span.begin && index < span.end)	80✔
28	return false;	12✔
29	}
30	return true;	92✔
31
32	}
33
34	function findSubstring(sequence, substring, spansBySentencePos) {
35	for (let i = 0; i < sequence.length - substring.length + 1; i++) {	92✔
36	let found = true;	472✔
37
38	for (let j = 0; j < substring.length; j++) {	472✔
39	if (sequence[i+j] !== substring[j]) {	552✔
40	found = false;	368✔
41	break;	368✔
42	}
43	}
44	if (found && check_range(i, spansBySentencePos))	472✔
45	return i;	92✔
46	}
47	return -1;	×
48	}
49
50
51	function findSpanType(program, begin_index, end_index) {
52	let spanType;
53	if (begin_index > 1 && program[begin_index-2] === 'location:') {	92✔
54	spanType = 'LOCATION';	4✔
55	} else if (program[begin_index-2] === '==' && program[begin_index-3].endsWith('Number')){	88!
NEW 56	spanType = 'NUMBER';	×
57	} else if (program[begin_index-2] === '==' && program[begin_index-3].includes('phone_number')){	88!
NEW 58	spanType = 'PHONE_NUMBER';	×
59	} else if (end_index === program.length - 1 \|\| !program[end_index+1].startsWith('^^')) {	88✔
60	spanType = 'QUOTED_STRING';	80✔
61	} else {
62	switch (program[end_index+1]) {	8!
63	case '^^tt:hashtag':
64	spanType = 'HASHTAG';	×
65	break;	×
66	case '^^tt:username':
67	spanType = 'USERNAME';	4✔
68	break;	4✔
69	default:
70	spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);	4✔
71	}
72	end_index++;	8✔
73	}
74	return [spanType, end_index];	92✔
75	}
76
77
78	function createProgram(program, spansByProgramPos, entityRemap) {
79	let in_string = false;	20✔
80	let newProgram = [];	20✔
81	let programSpanIndex = 0;	20✔
82	for (let i = 0; i < program.length; i++) {	20✔
83	let token = program[i];	532✔
84	if (token === '"') {	532✔
85	in_string = !in_string;	92✔
86	if (in_string)	92✔
87	continue;	46✔
88	const currentSpan = spansByProgramPos[programSpanIndex];	46✔
89	try{	46✔
90	assert(currentSpan.mapTo);	46✔
91	}
92	catch (e) {
93	console.log('error!');	×
94	console.log(program.join(' '));	×
95	}
96
97	newProgram.push(currentSpan.mapTo);	46✔
98	programSpanIndex++;	46✔
99	continue;	46✔
100	}
101	if (in_string) continue;	440✔
102
103	if (token === 'location:' \|\| token.startsWith('^^')) {	354✔
104	continue;	6✔
105	} else if (ENTITY_MATCH_REGEX.test(token)) {	348!
106	assert(entityRemap[token]);	×
107	newProgram.push(entityRemap[token]);	×
108	} else {
109	newProgram.push(token);	348✔
110	}
111	}
112	return newProgram;	20✔
113	}
114
115	function qpisSentence(sentence, spansBySentencePos) {
116
117	let current_span_idx = 0;	20✔
118	let current_span = spansBySentencePos[0];	20✔
119
120	let newSentence = [];	20✔
121	let i = 0;	20✔
122	let in_string = false;	20✔
123	while (i < sentence.length) {	20✔
124	let word = sentence[i];	302✔
125	if (current_span === null \|\| i < current_span.begin) {	302✔
126	newSentence.push(word);	174✔
127	i += 1;	174✔
128	} else if (i >= current_span.end) {	128✔
129	newSentence.push('"');	42✔
130	in_string = false;	42✔
131	current_span_idx += 1;	42✔
132	current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;	42✔
133	} else if (i === current_span.begin \|\| in_string) {	86!
134	if (i === current_span.begin)	86✔
135	newSentence.push('"');	46✔
136	newSentence.push(word);	86✔
137	in_string = true;	86✔
138	i += 1;	86✔
139
140	} else {
141	i += 1;	×
142	}
143	}
144	if (in_string)	20✔
145	newSentence.push('"');	4✔
146	return newSentence;	20✔
147
148	}
149
150	function createSentence(sentence, spansBySentencePos) {
151
152	let current_span_idx = 0;	20✔
153	let current_span = spansBySentencePos[0];	20✔
154
155	let newSentence = [];	20✔
156	let entityNumbers = {};	20✔
157	function getEntityNumber(entityType) {
158	let nextId = (entityNumbers[entityType] \|\| 0);	46✔
159	entityNumbers[entityType] = nextId + 1;	46✔
160	return String(nextId);	46✔
161	}
162	let entityRemap = {};	20✔
163
164	let i = 0;	20✔
165	while (i < sentence.length) {	20✔
166	let word = sentence[i];	302✔
167	if (current_span === null \|\| i < current_span.begin) {	302✔
168	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	174✔
169	if (entityMatch !== null) {	174!
170	// input sentence contains entities
171	const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);	×
172	entityRemap[word] = newEntity;	×
173	newSentence.push(newEntity);	×
174	} else {
175	newSentence.push(word);	174✔
176	}
177	i += 1;	174✔
178	} else if (i === current_span.begin) {	128✔
179	const newEntity = current_span.type + '_' + getEntityNumber(current_span.type);	46✔
180	current_span.mapTo = newEntity;	46✔
181	newSentence.push(newEntity);	46✔
182	i += 1;	46✔
183	} else if (i >= current_span.end) {	82✔
184	current_span_idx += 1;	42✔
185	current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;	42✔
186	} else {
187	i += 1;	40✔
188	}
189	}
190
191	return [newSentence, entityRemap];	20✔
192
193	}
194
195	function sortWithIndeces(toSort, sort_func) {
196	let toSort_new = [];	48✔
197	for (let i = 0; i < toSort.length; i++)	48✔
198	toSort_new[i] = [toSort[i], i];	92✔
199
200	toSort_new.sort(sort_func);	48✔
201	let sortIndices = [];	48✔
202	for (let j = 0; j < toSort_new.length; j++) {	48✔
203	sortIndices.push(toSort_new[j][1]);	92✔
204	toSort[j] = toSort_new[j][0];	92✔
205	}
206	return sortIndices;	48✔
207	}
208
209	function getProgSpans(program, is_quoted) {
210	let in_string = false;	48✔
211	let begin_index = null;	48✔
212	let end_index = null;	48✔
213	let all_prog_spans = [];	48✔
214	if (is_quoted) {	48!
NEW 215	for (let i = 0; i < program.length; i++) {	×
NEW 216	let token = program[i];	×
NEW 217	if (ENTITY_MATCH_REGEX.test(token)){	×
NEW 218	begin_index = i;	×
NEW 219	end_index = begin_index + 1;	×
UNCOV 220	let prog_span = {begin: begin_index, end: end_index};	×
UNCOV 221	all_prog_spans.push(prog_span);	×
222	}
223	}
224
225	} else {
226	for (let i = 0; i < program.length; i++) {	48✔
227	let token = program[i];	1,200✔
228	if (token === '"') {	1,200✔
229	in_string = !in_string;	184✔
230	if (in_string) {	184✔
231	begin_index = i + 1;	92✔
232	} else {
233	end_index = i;	92✔
234	let prog_span = {begin: begin_index, end: end_index};	92✔
235	all_prog_spans.push(prog_span);	92✔
236	}
237	}
238	}
239	}
240
241
242
243	// sort params based on length so that longer phrases get matched sooner
244	let sort_func = function (a, b) {	48✔
245	const {begin:abegin, end:aend} = a[0];	64✔
246	const {begin:bbegin, end:bend} = b[0];	64✔
247	return (bend - bbegin) - (aend - abegin);	64✔
248	};
249
250	// sort array in-place and return sorted indices
251	let sortIndices = sortWithIndeces(all_prog_spans, sort_func);	48✔
252
253	return [all_prog_spans, sortIndices];	48✔
254	}
255
256
257	function findSpanPositions(id, sentence, program, is_quoted) {
258	const spansBySentencePos = [];	48✔
259	const spansByProgramPos = [];	48✔
260
261	const [all_prog_spans_sorted, sortIndices] = getProgSpans(program, is_quoted);	48✔
262
263	for (let i = 0; i < all_prog_spans_sorted.length; i++) {	48✔
264	const prog_span = all_prog_spans_sorted[i];	92✔
265	let [begin_index, end_index] = [prog_span.begin, prog_span.end];	92✔
266	const substring = program.slice(begin_index, end_index);	92✔
267	const idx = findSubstring(sentence, substring, spansBySentencePos);	92✔
268	if (idx < 0){	92!
269	console.log(program.join(' '));	×
270	console.error('*Error: Program contains some parameters that are not present in the sentence*');	×
271	throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);	×
272
273	}
274
275	const spanBegin = idx;	92✔
276	const spanEnd = idx + end_index - begin_index;	92✔
277
278	let spanType = findSpanType(program, begin_index, end_index)[0];	92✔
279
280	const span = {begin: spanBegin, end: spanEnd, type: spanType, mapTo: undefined};	92✔
281	spansBySentencePos.push(span);	92✔
282	spansByProgramPos.push(span);	92✔
283
284	}
285
286	return [spansBySentencePos, spansByProgramPos, sortIndices];	48✔
287	}
288
289	function requoteSentence(id, sentence, program, mode, is_quoted) {
290	sentence = sentence.split(' ');	48✔
291	program = program.split(' ');	48✔
292
293	let [spansBySentencePos, spansByProgramPosSorted, sortIndices] = findSpanPositions(id, sentence, program, is_quoted);	48✔
294
295	if (spansBySentencePos.length === 0)	48✔
296	return [sentence.join(' '), program.join(' ')];	8✔
297
298	// revert back the order after matching is done
299	let spansByProgramPos = [];	40✔
300	for (let i = 0; i < sortIndices.length; i++)	40✔
301	spansByProgramPos[sortIndices[i]] = spansByProgramPosSorted[i];	92✔
302
303	spansBySentencePos.sort((a, b) => {	40✔
304	const {begin:abegin, end:aend} = a;	66✔
305	const {begin:bbegin, end:bend} = b;	66✔
306	if (abegin < bbegin)	66✔
307	return -1;	32✔
308	if (bbegin < abegin)	34!
309	return +1;	34✔
310	if (aend < bend)	×
311	return -1;	×
312	if (bend < aend)	×
313	return +1;	×
314	return 0;	×
315	});
316
317	let newSentence, newProgram, entityRemap;
318
319	if (mode === 'replace'){	40✔
320	[newSentence, entityRemap] = createSentence(sentence, spansBySentencePos);	20✔
321	newProgram = createProgram(program, spansByProgramPos, entityRemap);	20✔
322	} else if (mode === 'qpis') {	20!
323	newSentence = qpisSentence(sentence, spansBySentencePos);	20✔
324	newProgram = program;	20✔
325	}
326
327	return [newSentence.join(' '), newProgram.join(' ')];	40✔
328	}
329
330
331	module.exports = {	46✔
332	initArgparse(subparsers) {
333	const parser = subparsers.addParser('requote', {	46✔
334	addHelp: true,
335	description: "Requote a dataset."
336	});
337	parser.addArgument(['-o', '--output'], {	46✔
338	required: true,
339	type: fs.createWriteStream
340	});
341	parser.addArgument('--contextual', {	46✔
342	nargs: 0,
343	action: 'storeTrue',
344	help: 'Process a contextual dataset.',
345	defaultValue: false
346	});
347	parser.addArgument('--mode', {	46✔
348	type: String,
349	help: 'Type of requoting (replace with placeholder, or just add quotation marks around params)',
350	choices: ['replace', 'qpis'],
351	defaultValue: 'replace'
352	});
353	parser.addArgument('--is_quoted', {	46✔
354	action: 'storeTrue',
355	help: 'The input dataset is already quoted. Pass this to qpis a quoted dataset',
356	defaultValue: false
357	});
358	parser.addArgument('input_file', {	46✔
359	nargs: '+',
360	type: maybeCreateReadStream,
361	help: 'Input datasets to evaluate (in TSV format); use - for standard input'
362	});
363	},
364
365	async execute(args) {
366	readAllLines(args.input_file)	4✔
367	.pipe(new DatasetParser({ contextual: args.contextual }))
368	.pipe(new Stream.Transform({
369	objectMode: true,
370
371	transform(ex, encoding, callback) {
372	try {	48✔
373	const [newSentence, newProgram] = requoteSentence(ex.id, ex.preprocessed, ex.target_code, args.mode, args.is_quoted);	48✔
374	ex.preprocessed = newSentence;	48✔
375	ex.target_code = newProgram;	48✔
376	callback(null, ex);	48✔
377	} catch(e) {
378	console.error(`Failed to requote`);	×
379	console.error(ex.preprocessed);	×
380	console.error(ex.target_code);	×
381	callback(e);	×
382	}
383	},
384
385	flush(callback) {
386	process.nextTick(callback);	4✔
387	}
388	}))
389	.pipe(new DatasetStringifier())
390	.pipe(args.output);
391
392	await StreamUtils.waitFinish(args.output);	4✔
393	}
394	};

stanford-oval / genie-toolkit / 710

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous