stanford-oval / genie-toolkit / 882

pending completion

Build # 882

Build Type

Pull #185

travis-ci-com

Committed by

web-flow

Commit Message

Merge fa73f1c9b into b0d881655

Pull Request Pull Request #185: Rearrange some code to simplify review

Run Details

2522 of 8677 branches covered (29.07%)

Branch coverage included in aggregate %.

2901 of 5099 new or added lines in 60 files covered. (56.89%)

2 existing lines in 1 file now uncovered.

6929 of 16432 relevant lines covered (42.17%)

87427.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.77

/tool/lib/mturk-parsers.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Silei Xu <silei@cs.stanford.edu>
//         Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const Stream = require('stream');

const { ParaphraseValidator } = require('../../lib/dataset-tools/mturk/validator');

// Parse the raw output of Amazon MTurk paraphrasing into easier to handle objects
//
// Input:
//  - one row per task submission
//    formatted as per MTurk results
//
// Output:
//  - one row per paraphrase, each with:
//    - id
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrase
class ParaphrasingParser extends Stream.Transform {
    constructor(options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._sentencesPerTask = options.sentencesPerTask;
        this._paraphrasesPerSentence = options.paraphrasesPerSentence;
        this._contextual = options.contextual;

        this._id = 0;
    }

    _transform(row, encoding, callback) {
        for (let i = 0; i < this._sentencesPerTask; i++) {
            const target_code = row[`Input.thingtalk${i+1}`];
            const synthetic = row[`Input.sentence${i+1}`];
            const synthetic_id = row[`Input.id${i+1}`];

            let context, context_utterance, assistant_action;
            if (this._contextual) {
                context = row[`Input.context${i+1}`];
                context_utterance = row[`Input.context_utterance${i+1}`];
                assistant_action = row[`Input.assistant_action${i+1}`];
            }

            for (let j = 0; j < this._paraphrasesPerSentence; j++) {
                const paraphrase = row[`Answer.Paraphrase${i+1}-${j+1}`];
                const id = (this._contextual ? 'C' : '') + this._id++;
                if (this._contextual) {
                    this.push({
                        id, synthetic_id, synthetic,
                        context, context_utterance, assistant_action,
                        target_code, paraphrase
                    });
                } else {
                    this.push({
                        id, synthetic_id, synthetic, target_code, paraphrase
                    });
                }
            }
        }
        callback();
    }

    _flush(callback) {
        process.nextTick(callback);
    }
}

// Accumulate all paraphrases of the same synthetic
//
// Input:
//  - one row per paraphrase, each with:
//    - id
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrase
//
// Output:
//  - one row per synthetic, each with:
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrases: array of { id, paraphrase }
class ParaphrasingAccumulator extends Stream.Transform {
    constructor(targetSize) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._targetSize = targetSize;

        this._synthetics = new Map;
        this._buffers = new Map;
    }

    _transform(row, encoding, callback) {
        const synthetic_id = row.synthetic_id;

        let buffer = this._buffers.get(synthetic_id);
        if (!buffer) {
            this._synthetics.set(synthetic_id, row);
            this._buffers.set(synthetic_id, buffer = []);
        }

        buffer.push({
            id: row.id,
            paraphrase: row.paraphrase
        });

        if (buffer.length >= this._totalParaphrasesPerSentence) {
            this.push({
                synthetic_id,
                synthetic: row.synthetic,
                target_code: row.target_code,
                paraphrases: buffer
            });
            this._buffers.delete(synthetic_id);
        }
        callback();
    }

    _flush(callback) {
        for (let [synthetic_id, buffer] of this._buffers) {
            let row = this._synthetics.get(synthetic_id);
            this.push({
                synthetic_id,
                synthetic: row.synthetic,
                target_code: row.target_code,
                paraphrases: buffer
            });
        }

        this._buffers.clear();
        this._synthetics.clear();
        callback();
    }
}

// Parse the raw output of Amazon MTurk validation into easier to handle objects
//
// Input:
//  - one row per task submission
//    formatted as per MTurk results
//
// Output:
//  - one row per paraphrase, each with:
//    - id
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrase
//    - vote
class ValidationParser extends Stream.Transform {
    constructor(options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._sentencesPerTask = options.sentencesPerTask;
        this._targetSize = options.targetSize;
        this._skipRejected = options.skipRejected;

        this._id = 0;
    }

    _transform(row, encoding, callback) {
        if (this._skipRejected && row['Reject']) {
            callback();
            return;
        }

        for (let i = 0; i < this._sentencesPerTask; i++) {
            const target_code = row[`Input.thingtalk${i+1}`];
            const synthetic = row[`Input.sentence${i+1}`];
            const synthetic_id = row[`Input.id${i+1}`];

            // + 2 to account for same/different paraphrases (sanity checks)
            // mixed in with the real data
            for (let j = 0; j < this._targetSize + 2; j++) {
                const paraphrase = row[`Input.paraphrase${i+1}-${j+1}`];
                const id = row[`Input.id${i+1}-${j+1}`];
                if (id === '-same' || id === '-different')
                    continue;

                const vote = row[`Answer.${i+1}-${j+1}`];
                this.push({
                    id, synthetic_id, synthetic, target_code, paraphrase, vote
                });
            }
        }
        callback();
    }

    _flush(callback) {
        process.nextTick(callback);
    }
}

// Count all validation votes of the same paraphrase
//
// Input:
//  - one row per validation task, each with:
//    - id
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrase
//    - vote
//
// Output:
//  - one row per paraphrase, each with:
//    - id
//    - synthetic_id
//    - synthetic
//    - target_code
//    - paraphrase
//    - same_count
//    - diff_count
class ValidationCounter extends Stream.Transform {
    constructor(options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._targetNumVotes = options.targetNumVotes;
        this._buffer = new Map;
    }

    _transform(row, encoding, callback) {
        const id = row.id;

        let count = this._buffer.get(id);
        if (!count) {
            this._buffer.set(id, count = {
                id,
                synthetic_id: row.synthetic_id,
                synthetic: row.synthetic,
                target_code: row.target_code,
                paraphrase: row.paraphrase,
                same_count: 0,
                diff_count: 0
            });
        }

        if (row.vote === 'same')
            count.same_count ++;
        else
            count.diff_count ++;

        if (count.same_count + count.diff_count >= this._targetNumVotes) {
            this.push(count);
            this._buffer.delete(id);
        }
        callback();
    }

    _flush(callback) {
        for (let count of this._buffer.values())
            this.push(count);

        this._buffer.clear();
        callback();
    }
}

class ValidationRejecter extends Stream.Transform {
    constructor(options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._sentencesPerTask = options.sentencesPerTask;
    }

    _transform(row, encoding, callback) {
        let incorrect = 0;
        for (let i = 0; i < this._sentencesPerTask; i++) {
            const indexSame = row[`Input.index_same${i+1}`];
            const indexDifferent = row[`Input.index_diff${i+1}`];
            const answerSame = row[`Answer.${i+1}-${indexSame}`];
            const answerDifferent = row[`Answer.${i+1}-${indexDifferent}`];
            if (answerSame !== 'same')
                incorrect++;
            if (answerDifferent !== 'different')
                incorrect++;
        }

        if (incorrect >= 2) {
            row['Approve'] = '';
            row['Reject'] = '2 or more mistakes in the sanity checks hidden among the questions.';
        } else {
            row['Approve'] = 'x';
            row['Reject'] = '';
        }

        callback(null, row);
    }

    _flush(callback) {
        process.nextTick(callback);
    }
}

class ParaphrasingRejecter extends Stream.Transform {
    constructor(schemaRetriever, tokenizer, options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });

        this._schemas = schemaRetriever;
        this._tokenizer = tokenizer;

        this._locale = options.locale;
        this._sentencesPerTask = options.sentencesPerTask;
        this._paraphrasesPerSentence = options.paraphrasesPerSentence;
        this._contextual = options.contextual;

        this._counter = {};

        this._id = 0;
    }

    async _validate(paraobj) {
        const paraphrase = new ParaphraseValidator(this._schemas, this._tokenizer, this._locale,
            paraobj, this._counter);

        try {
            await paraphrase.clean();
            if (paraphrase.isValid())
                return paraobj;
            else
                return null;
        } catch(e) {
            console.error(`Failed paraphrase ${paraobj.id} (${paraobj.synthetic_id}): ${e.message}`);
            return null;
        }
    }

    async _doTransform(row) {
        const minibatch = [];

        for (let i = 0; i < this._sentencesPerTask; i++) {
            const target_code = row[`Input.thingtalk${i+1}`];
            const synthetic = row[`Input.sentence${i+1}`];
            const synthetic_id = row[`Input.id${i+1}`];

            let context, context_utterance, assistant_action;
            if (this._contextual) {
                context = row[`Input.context${i+1}`];
                context_utterance = row[`Input.context_utterance${i+1}`];
                assistant_action = row[`Input.assistant_action${i+1}`];
            }

            for (let j = 0; j < this._paraphrasesPerSentence; j++) {
                const paraphrase = row[`Answer.Paraphrase${i+1}-${j+1}`];
                const id = (this._contextual ? 'C' : '') + this._id++;

                let paraobj;
                if (this._contextual) {
                    paraobj = {
                        id, synthetic_id, synthetic,
                        context, context_utterance, assistant_action,
                        target_code, paraphrase
                    };
                } else {
                    paraobj = {
                        id, synthetic_id, synthetic, target_code, paraphrase
                    };
                }
                minibatch.push(paraobj);
            }
        }

        const validated = (await Promise.all(minibatch.map((paraobj) => {
            return this._validate(paraobj);
        }))).filter((paraobj) => paraobj !== null);

        if (validated.length < this._sentencesPerTask * this._paraphrasesPerSentence - 2) {
            row['Approve'] = '';
            row['Reject'] = `Failed to give reasonable result or failed to follow the instruction in at least 2 of ${this._sentencesPerTask * this._paraphrasesPerSentence} paraphrases`;
        } else {
            row['Approve'] = 'x';
            row['Reject'] = '';
        }
        return row;
    }

    _transform(row, encoding, callback) {
        this._doTransform(row).then(
            (row) => callback(null, row),
            (err) => callback(err)
        );
    }

    _flush(callback) {
        process.nextTick(callback);
    }
}

module.exports = {
    ParaphrasingParser,
    ParaphrasingAccumulator,
    ValidationParser,
    ValidationCounter,
    ValidationRejecter,
    ParaphrasingRejecter
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Silei Xu <silei@cs.stanford.edu>
8	// Giovanni Campagna <gcampagn@cs.stanford.edu>
9	//
10	// See COPYING for details
11	"use strict";
12
13	const Stream = require('stream');	52✔
14
15	const { ParaphraseValidator } = require('../../lib/dataset-tools/mturk/validator');	52✔
16
17	// Parse the raw output of Amazon MTurk paraphrasing into easier to handle objects
18	//
19	// Input:
20	// - one row per task submission
21	// formatted as per MTurk results
22	//
23	// Output:
24	// - one row per paraphrase, each with:
25	// - id
26	// - synthetic_id
27	// - synthetic
28	// - target_code
29	// - paraphrase
30	class ParaphrasingParser extends Stream.Transform {
31	constructor(options) {
32	super({	6✔
33	readableObjectMode: true,
34	writableObjectMode: true
35	});
36
37	this._sentencesPerTask = options.sentencesPerTask;	6✔
38	this._paraphrasesPerSentence = options.paraphrasesPerSentence;	6✔
39	this._contextual = options.contextual;	6✔
40
41	this._id = 0;	6✔
42	}
43
44	_transform(row, encoding, callback) {
45	for (let i = 0; i < this._sentencesPerTask; i++) {	36✔
46	const target_code = row[`Input.thingtalk${i+1}`];	144✔
47	const synthetic = row[`Input.sentence${i+1}`];	144✔
48	const synthetic_id = row[`Input.id${i+1}`];	144✔
49
50	let context, context_utterance, assistant_action;
51	if (this._contextual) {	144!
52	context = row[`Input.context${i+1}`];	×
53	context_utterance = row[`Input.context_utterance${i+1}`];	×
54	assistant_action = row[`Input.assistant_action${i+1}`];	×
55	}
56
57	for (let j = 0; j < this._paraphrasesPerSentence; j++) {	144✔
58	const paraphrase = row[`Answer.Paraphrase${i+1}-${j+1}`];	288✔
59	const id = (this._contextual ? 'C' : '') + this._id++;	288!
60	if (this._contextual) {	288!
61	this.push({	×
62	id, synthetic_id, synthetic,
63	context, context_utterance, assistant_action,
64	target_code, paraphrase
65	});
66	} else {
67	this.push({	288✔
68	id, synthetic_id, synthetic, target_code, paraphrase
69	});
70	}
71	}
72	}
73	callback();	36✔
74	}
75
76	_flush(callback) {
77	process.nextTick(callback);	6✔
78	}
79	}
80
81	// Accumulate all paraphrases of the same synthetic
82	//
83	// Input:
84	// - one row per paraphrase, each with:
85	// - id
86	// - synthetic_id
87	// - synthetic
88	// - target_code
89	// - paraphrase
90	//
91	// Output:
92	// - one row per synthetic, each with:
93	// - synthetic_id
94	// - synthetic
95	// - target_code
96	// - paraphrases: array of { id, paraphrase }
97	class ParaphrasingAccumulator extends Stream.Transform {
98	constructor(targetSize) {
99	super({	2✔
100	readableObjectMode: true,
101	writableObjectMode: true
102	});
103
104	this._targetSize = targetSize;	2✔
105
106	this._synthetics = new Map;	2✔
107	this._buffers = new Map;	2✔
108	}
109
110	_transform(row, encoding, callback) {
111	const synthetic_id = row.synthetic_id;	80✔
112
113	let buffer = this._buffers.get(synthetic_id);	80✔
114	if (!buffer) {	80✔
115	this._synthetics.set(synthetic_id, row);	16✔
116	this._buffers.set(synthetic_id, buffer = []);	16✔
117	}
118
119	buffer.push({	80✔
120	id: row.id,
121	paraphrase: row.paraphrase
122	});
123
124	if (buffer.length >= this._totalParaphrasesPerSentence) {	80!
125	this.push({	×
126	synthetic_id,
127	synthetic: row.synthetic,
128	target_code: row.target_code,
129	paraphrases: buffer
130	});
131	this._buffers.delete(synthetic_id);	×
132	}
133	callback();	80✔
134	}
135
136	_flush(callback) {
137	for (let [synthetic_id, buffer] of this._buffers) {	2✔
138	let row = this._synthetics.get(synthetic_id);	16✔
139	this.push({	16✔
140	synthetic_id,
141	synthetic: row.synthetic,
142	target_code: row.target_code,
143	paraphrases: buffer
144	});
145	}
146
147	this._buffers.clear();	2✔
148	this._synthetics.clear();	2✔
149	callback();	2✔
150	}
151	}
152
153	// Parse the raw output of Amazon MTurk validation into easier to handle objects
154	//
155	// Input:
156	// - one row per task submission
157	// formatted as per MTurk results
158	//
159	// Output:
160	// - one row per paraphrase, each with:
161	// - id
162	// - synthetic_id
163	// - synthetic
164	// - target_code
165	// - paraphrase
166	// - vote
167	class ValidationParser extends Stream.Transform {
168	constructor(options) {
169	super({	2✔
170	readableObjectMode: true,
171	writableObjectMode: true
172	});
173
174	this._sentencesPerTask = options.sentencesPerTask;	2✔
175	this._targetSize = options.targetSize;	2✔
176	this._skipRejected = options.skipRejected;	2✔
177
178	this._id = 0;	2✔
179	}
180
181	_transform(row, encoding, callback) {
182	if (this._skipRejected && row['Reject']) {	8✔
183	callback();	2✔
184	return;	2✔
185	}
186
187	for (let i = 0; i < this._sentencesPerTask; i++) {	6✔
188	const target_code = row[`Input.thingtalk${i+1}`];	24✔
189	const synthetic = row[`Input.sentence${i+1}`];	24✔
190	const synthetic_id = row[`Input.id${i+1}`];	24✔
191
192	// + 2 to account for same/different paraphrases (sanity checks)
193	// mixed in with the real data
194	for (let j = 0; j < this._targetSize + 2; j++) {	24✔
195	const paraphrase = row[`Input.paraphrase${i+1}-${j+1}`];	192✔
196	const id = row[`Input.id${i+1}-${j+1}`];	192✔
197	if (id === '-same' \|\| id === '-different')	192✔
198	continue;	48✔
199
200	const vote = row[`Answer.${i+1}-${j+1}`];	144✔
201	this.push({	144✔
202	id, synthetic_id, synthetic, target_code, paraphrase, vote
203	});
204	}
205	}
206	callback();	6✔
207	}
208
209	_flush(callback) {
210	process.nextTick(callback);	2✔
211	}
212	}
213
214	// Count all validation votes of the same paraphrase
215	//
216	// Input:
217	// - one row per validation task, each with:
218	// - id
219	// - synthetic_id
220	// - synthetic
221	// - target_code
222	// - paraphrase
223	// - vote
224	//
225	// Output:
226	// - one row per paraphrase, each with:
227	// - id
228	// - synthetic_id
229	// - synthetic
230	// - target_code
231	// - paraphrase
232	// - same_count
233	// - diff_count
234	class ValidationCounter extends Stream.Transform {
235	constructor(options) {
236	super({	2✔
237	readableObjectMode: true,
238	writableObjectMode: true
239	});
240
241	this._targetNumVotes = options.targetNumVotes;	2✔
242	this._buffer = new Map;	2✔
243	}
244
245	_transform(row, encoding, callback) {
246	const id = row.id;	144✔
247
248	let count = this._buffer.get(id);	144✔
249	if (!count) {	144✔
250	this._buffer.set(id, count = {	48✔
251	id,
252	synthetic_id: row.synthetic_id,
253	synthetic: row.synthetic,
254	target_code: row.target_code,
255	paraphrase: row.paraphrase,
256	same_count: 0,
257	diff_count: 0
258	});
259	}
260
261	if (row.vote === 'same')	144✔
262	count.same_count ++;	132✔
263	else
264	count.diff_count ++;	12✔
265
266	if (count.same_count + count.diff_count >= this._targetNumVotes) {	144✔
267	this.push(count);	48✔
268	this._buffer.delete(id);	48✔
269	}
270	callback();	144✔
271	}
272
273	_flush(callback) {
274	for (let count of this._buffer.values())	2✔
275	this.push(count);	×
276
277	this._buffer.clear();	2✔
278	callback();	2✔
279	}
280	}
281
282	class ValidationRejecter extends Stream.Transform {
283	constructor(options) {
284	super({	2✔
285	readableObjectMode: true,
286	writableObjectMode: true
287	});
288
289	this._sentencesPerTask = options.sentencesPerTask;	2✔
290	}
291
292	_transform(row, encoding, callback) {
293	let incorrect = 0;	8✔
294	for (let i = 0; i < this._sentencesPerTask; i++) {	8✔
295	const indexSame = row[`Input.index_same${i+1}`];	32✔
296	const indexDifferent = row[`Input.index_diff${i+1}`];	32✔
297	const answerSame = row[`Answer.${i+1}-${indexSame}`];	32✔
298	const answerDifferent = row[`Answer.${i+1}-${indexDifferent}`];	32✔
299	if (answerSame !== 'same')	32!
300	incorrect++;	×
301	if (answerDifferent !== 'different')	32✔
302	incorrect++;	8✔
303	}
304
305	if (incorrect >= 2) {	8✔
306	row['Approve'] = '';	2✔
307	row['Reject'] = '2 or more mistakes in the sanity checks hidden among the questions.';	2✔
308	} else {
309	row['Approve'] = 'x';	6✔
310	row['Reject'] = '';	6✔
311	}
312
313	callback(null, row);	8✔
314	}
315
316	_flush(callback) {
317	process.nextTick(callback);	2✔
318	}
319	}
320
321	class ParaphrasingRejecter extends Stream.Transform {
322	constructor(schemaRetriever, tokenizer, options) {
323	super({	4✔
324	readableObjectMode: true,
325	writableObjectMode: true
326	});
327
328	this._schemas = schemaRetriever;	4✔
329	this._tokenizer = tokenizer;	4✔
330
331	this._locale = options.locale;	4✔
332	this._sentencesPerTask = options.sentencesPerTask;	4✔
333	this._paraphrasesPerSentence = options.paraphrasesPerSentence;	4✔
334	this._contextual = options.contextual;	4✔
335
336	this._counter = {};	4✔
337
338	this._id = 0;	4✔
339	}
340
341	async _validate(paraobj) {
342	const paraphrase = new ParaphraseValidator(this._schemas, this._tokenizer, this._locale,	192✔
343	paraobj, this._counter);
344
345	try {	192✔
346	await paraphrase.clean();	192✔
347	if (paraphrase.isValid())	192✔
348	return paraobj;	160✔
349	else
350	return null;	32✔
351	} catch(e) {
UNCOV 352	console.error(`Failed paraphrase ${paraobj.id} (${paraobj.synthetic_id}): ${e.message}`);	×
UNCOV 353	return null;	×
354	}
355	}
356
357	async _doTransform(row) {
358	const minibatch = [];	24✔
359
360	for (let i = 0; i < this._sentencesPerTask; i++) {	24✔
361	const target_code = row[`Input.thingtalk${i+1}`];	96✔
362	const synthetic = row[`Input.sentence${i+1}`];	96✔
363	const synthetic_id = row[`Input.id${i+1}`];	96✔
364
365	let context, context_utterance, assistant_action;
366	if (this._contextual) {	96!
367	context = row[`Input.context${i+1}`];	×
368	context_utterance = row[`Input.context_utterance${i+1}`];	×
369	assistant_action = row[`Input.assistant_action${i+1}`];	×
370	}
371
372	for (let j = 0; j < this._paraphrasesPerSentence; j++) {	96✔
373	const paraphrase = row[`Answer.Paraphrase${i+1}-${j+1}`];	192✔
374	const id = (this._contextual ? 'C' : '') + this._id++;	192!
375
376	let paraobj;
377	if (this._contextual) {	192!
378	paraobj = {	×
379	id, synthetic_id, synthetic,
380	context, context_utterance, assistant_action,
381	target_code, paraphrase
382	};
383	} else {
384	paraobj = {	192✔
385	id, synthetic_id, synthetic, target_code, paraphrase
386	};
387	}
388	minibatch.push(paraobj);	192✔
389	}
390	}
391
392	const validated = (await Promise.all(minibatch.map((paraobj) => {	24✔
393	return this._validate(paraobj);	192✔
394	}))).filter((paraobj) => paraobj !== null);	192✔
395
396	if (validated.length < this._sentencesPerTask * this._paraphrasesPerSentence - 2) {	24✔
397	row['Approve'] = '';	4✔
398	row['Reject'] = `Failed to give reasonable result or failed to follow the instruction in at least 2 of ${this._sentencesPerTask * this._paraphrasesPerSentence} paraphrases`;	4✔
399	} else {
400	row['Approve'] = 'x';	20✔
401	row['Reject'] = '';	20✔
402	}
403	return row;	24✔
404	}
405
406	_transform(row, encoding, callback) {
407	this._doTransform(row).then(	24✔
408	(row) => callback(null, row),	24✔
409	(err) => callback(err)	×
410	);
411	}
412
413	_flush(callback) {
414	process.nextTick(callback);	4✔
415	}
416	}
417
418	module.exports = {	52✔
419	ParaphrasingParser,
420	ParaphrasingAccumulator,
421	ValidationParser,
422	ValidationCounter,
423	ValidationRejecter,
424	ParaphrasingRejecter
425	};

stanford-oval / genie-toolkit / 882

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous