stanford-oval / genie-toolkit / 591

pending completion

Build # 591

Build Type

Pull #124

travis-ci-com

Committed by

web-flow

Commit Message

Fix lint

Pull Request Pull Request #124: Merge schema.org to master

Run Details

1684 of 4124 branches covered (40.83%)

Branch coverage included in aggregate %.

333 of 1694 new or added lines in 26 files covered. (19.66%)

4 existing lines in 4 files now uncovered.

4269 of 8267 relevant lines covered (51.64%)

555172.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.22

/tool/requote.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const fs = require('fs');
const Stream = require('stream');
const assert = require('assert');

const { DatasetParser, DatasetStringifier } = require('../lib/dataset-parsers');
const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');
const StreamUtils = require('../lib/stream-utils');

function findSubstring(sequence, substring) {
    for (let i = 0; i < sequence.length - substring.length + 1; i++) {
        let found = true;
        for (let j = 0; j < substring.length; j++) {
            if (sequence[i+j] !== substring[j]) {
                found = false;
                break;
            }
        }
        if (found)
            return i;
    }
    return -1;
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;

function findSpanType(program, begin_index, end_index) {
    let spanType;
    if (begin_index > 1 && program[begin_index-2] === 'location:') {
        spanType = 'LOCATION';
    } else if (end_index === program.length - 1 || !program[end_index+1].startsWith('^^')) {
        spanType = 'QUOTED_STRING';
    } else {
        switch (program[end_index+1]) {
        case '^^tt:hashtag':
            spanType = 'HASHTAG';
            break;
        case '^^tt:username':
            spanType = 'USERNAME';
            break;
        default:
            spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);
        }
        end_index++;
    }
    return [spanType, end_index];
}

function requoteSentence(id, sentence, program) {
    sentence = sentence.split(' ');
    program = program.split(' ');

    const spansBySentencePos = [];
    const spansByProgramPos = [];

    let in_string = false;
    let begin_index = null;
    let end_index = null;
    for (let i = 0; i < program.length; i++) {
        let token = program[i];
        if (token === '"') {
            in_string = !in_string;
            if (in_string) {
                begin_index = i+1;
            } else {
                end_index = i;
                const substring = program.slice(begin_index, end_index);
                const idx = findSubstring(sentence, substring);
                if (idx < 0)
                    throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);

                const spanBegin = idx;
                const spanEnd = idx+end_index-begin_index;

                let spanType;
                [spanType, end_index] = findSpanType(program, begin_index, end_index);

                const span = { begin: spanBegin, end: spanEnd, type: spanType, mapTo: undefined };
                spansBySentencePos.push(span);
                spansByProgramPos.push(span);
                i = end_index;
            }
        }
    }

    if (spansBySentencePos.length === 0)
        return [sentence.join(' '), program.join(' ')];

    spansBySentencePos.sort((a, b) => {
        const {begin:abegin, end:aend} = a;
        const {begin:bbegin, end:bend} = b;
        if (abegin < bbegin)
            return -1;
        if (bbegin < abegin)
            return +1;
        if (aend < bend)
            return -1;
        if (bend < aend)
            return +1;
        return 0;
    });
    let current_span_idx = 0;
    let current_span = spansBySentencePos[0];

    let newSentence = [];
    let entityNumbers = {};
    function getEntityNumber(entityType) {
        let nextId = (entityNumbers[entityType] || 0);
        entityNumbers[entityType] = nextId + 1;
        return String(nextId);
    }
    let entityRemap = {};

    for (let i = 0; i < sentence.length; i++) {
        let word = sentence[i];
        if (current_span === null || i < current_span.begin) {
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null) {
                const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);
                entityRemap[word] = newEntity;
                newSentence.push(newEntity);
            } else {
                newSentence.push(word);
            }
        } else if (i === current_span.begin) {
            const newEntity = current_span.type + '_' + getEntityNumber(current_span.type);
            current_span.mapTo = newEntity;
            newSentence.push(newEntity);
        } else if (i >= current_span.end) {
            current_span_idx += 1;
            current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;

            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null) {
                const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);
                entityRemap[word] = newEntity;
                newSentence.push(newEntity);
            } else {
                newSentence.push(word);
            }
        }
    }

    let newProgram = [];
    let programSpanIndex = 0;
    for (let i = 0; i < program.length; i++) {
        let token = program[i];
        if (token === '"') {
            in_string = !in_string;
            if (in_string)
                continue;
            const currentSpan = spansByProgramPos[programSpanIndex];
            assert(currentSpan.mapTo);
            newProgram.push(currentSpan.mapTo);
            programSpanIndex ++;
            continue;
        }
        if (in_string) continue;

        if (token === 'location:' || token.startsWith('^^')) {
            continue;
        } else if (ENTITY_MATCH_REGEX.test(token)) {
            if (!entityRemap[token])
                console.error(id, token, program, sentence);
            assert(entityRemap[token]);
            newProgram.push(entityRemap[token]);
        } else {
            newProgram.push(token);
        }
    }

    return [newSentence.join(' '), newProgram.join(' ')];
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('requote', {
            addHelp: true,
            description: "Requote a dataset."
        });
        parser.addArgument(['-o', '--output'], {
            required: true,
            type: fs.createWriteStream
        });
        parser.addArgument('--contextual', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Process a contextual dataset.',
            defaultValue: false
        });
        parser.addArgument('input_file', {
            nargs: '+',
            type: maybeCreateReadStream,
            help: 'Input datasets to evaluate (in TSV format); use - for standard input'
        });
    },

    async execute(args) {
        readAllLines(args.input_file)
            .pipe(new DatasetParser({ contextual: args.contextual }))
            .pipe(new Stream.Transform({
                objectMode: true,

                transform(ex, encoding, callback) {
                    const [newSentence, newProgram] = requoteSentence(ex.id, ex.preprocessed, ex.target_code);
                    ex.preprocessed = newSentence;
                    ex.target_code = newProgram;
                    callback(null, ex);
                },

                flush(callback) {
                    process.nextTick(callback);
                }
            }))
            .pipe(new DatasetStringifier())
            .pipe(args.output);

        await StreamUtils.waitFinish(args.output);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const fs = require('fs');	52✔
13	const Stream = require('stream');	52✔
14	const assert = require('assert');	52✔
15
16	const { DatasetParser, DatasetStringifier } = require('../lib/dataset-parsers');	52✔
17	const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');	52✔
18	const StreamUtils = require('../lib/stream-utils');	52✔
19
20	function findSubstring(sequence, substring) {
21	for (let i = 0; i < sequence.length - substring.length + 1; i++) {	×
22	let found = true;	×
23	for (let j = 0; j < substring.length; j++) {	×
24	if (sequence[i+j] !== substring[j]) {	×
25	found = false;	×
26	break;	×
27	}
28	}
29	if (found)	×
30	return i;	×
31	}
32	return -1;	×
33	}
34
35	const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;	52✔
36
37	function findSpanType(program, begin_index, end_index) {
38	let spanType;
39	if (begin_index > 1 && program[begin_index-2] === 'location:') {	×
40	spanType = 'LOCATION';	×
41	} else if (end_index === program.length - 1 \|\| !program[end_index+1].startsWith('^^')) {	×
42	spanType = 'QUOTED_STRING';	×
43	} else {
44	switch (program[end_index+1]) {	×
45	case '^^tt:hashtag':
46	spanType = 'HASHTAG';	×
47	break;	×
48	case '^^tt:username':
49	spanType = 'USERNAME';	×
50	break;	×
51	default:
52	spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);	×
53	}
54	end_index++;	×
55	}
56	return [spanType, end_index];	×
57	}
58
59	function requoteSentence(id, sentence, program) {
60	sentence = sentence.split(' ');	×
61	program = program.split(' ');	×
62
63	const spansBySentencePos = [];	×
64	const spansByProgramPos = [];	×
65
66	let in_string = false;	×
67	let begin_index = null;	×
68	let end_index = null;	×
69	for (let i = 0; i < program.length; i++) {	×
70	let token = program[i];	×
71	if (token === '"') {	×
72	in_string = !in_string;	×
73	if (in_string) {	×
74	begin_index = i+1;	×
75	} else {
76	end_index = i;	×
77	const substring = program.slice(begin_index, end_index);	×
78	const idx = findSubstring(sentence, substring);	×
79	if (idx < 0)	×
80	throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);	×
81
82	const spanBegin = idx;	×
83	const spanEnd = idx+end_index-begin_index;	×
84
85	let spanType;
86	[spanType, end_index] = findSpanType(program, begin_index, end_index);	×
87
88	const span = { begin: spanBegin, end: spanEnd, type: spanType, mapTo: undefined };	×
89	spansBySentencePos.push(span);	×
90	spansByProgramPos.push(span);	×
91	i = end_index;	×
92	}
93	}
94	}
95
96	if (spansBySentencePos.length === 0)	×
97	return [sentence.join(' '), program.join(' ')];	×
98
99	spansBySentencePos.sort((a, b) => {	×
100	const {begin:abegin, end:aend} = a;	×
101	const {begin:bbegin, end:bend} = b;	×
102	if (abegin < bbegin)	×
103	return -1;	×
104	if (bbegin < abegin)	×
105	return +1;	×
106	if (aend < bend)	×
107	return -1;	×
108	if (bend < aend)	×
109	return +1;	×
110	return 0;	×
111	});
112	let current_span_idx = 0;	×
113	let current_span = spansBySentencePos[0];	×
114
115	let newSentence = [];	×
116	let entityNumbers = {};	×
117	function getEntityNumber(entityType) {
118	let nextId = (entityNumbers[entityType] \|\| 0);	×
119	entityNumbers[entityType] = nextId + 1;	×
120	return String(nextId);	×
121	}
122	let entityRemap = {};	×
123
124	for (let i = 0; i < sentence.length; i++) {	×
125	let word = sentence[i];	×
126	if (current_span === null \|\| i < current_span.begin) {	×
127	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	×
128	if (entityMatch !== null) {	×
129	const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);	×
130	entityRemap[word] = newEntity;	×
131	newSentence.push(newEntity);	×
132	} else {
133	newSentence.push(word);	×
134	}
135	} else if (i === current_span.begin) {	×
136	const newEntity = current_span.type + '_' + getEntityNumber(current_span.type);	×
137	current_span.mapTo = newEntity;	×
138	newSentence.push(newEntity);	×
139	} else if (i >= current_span.end) {	×
140	current_span_idx += 1;	×
141	current_span = current_span_idx < spansBySentencePos.length ? spansBySentencePos[current_span_idx] : null;	×
142
NEW 143	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	×
NEW 144	if (entityMatch !== null) {	×
NEW 145	const newEntity = entityMatch[1] + '_' + getEntityNumber(entityMatch[1]);	×
NEW 146	entityRemap[word] = newEntity;	×
NEW 147	newSentence.push(newEntity);	×
148	} else {
NEW 149	newSentence.push(word);	×
150	}
151	}
152	}
153
154	let newProgram = [];	×
155	let programSpanIndex = 0;	×
156	for (let i = 0; i < program.length; i++) {	×
157	let token = program[i];	×
158	if (token === '"') {	×
159	in_string = !in_string;	×
160	if (in_string)	×
161	continue;	×
162	const currentSpan = spansByProgramPos[programSpanIndex];	×
163	assert(currentSpan.mapTo);	×
164	newProgram.push(currentSpan.mapTo);	×
NEW 165	programSpanIndex ++;	×
UNCOV 166	continue;	×
167	}
168	if (in_string) continue;	×
169
170	if (token === 'location:' \|\| token.startsWith('^^')) {	×
171	continue;	×
172	} else if (ENTITY_MATCH_REGEX.test(token)) {	×
NEW 173	if (!entityRemap[token])	×
NEW 174	console.error(id, token, program, sentence);	×
175	assert(entityRemap[token]);	×
176	newProgram.push(entityRemap[token]);	×
177	} else {
178	newProgram.push(token);	×
179	}
180	}
181
182	return [newSentence.join(' '), newProgram.join(' ')];	×
183	}
184
185	module.exports = {	52✔
186	initArgparse(subparsers) {
187	const parser = subparsers.addParser('requote', {	52✔
188	addHelp: true,
189	description: "Requote a dataset."
190	});
191	parser.addArgument(['-o', '--output'], {	52✔
192	required: true,
193	type: fs.createWriteStream
194	});
195	parser.addArgument('--contextual', {	52✔
196	nargs: 0,
197	action: 'storeTrue',
198	help: 'Process a contextual dataset.',
199	defaultValue: false
200	});
201	parser.addArgument('input_file', {	52✔
202	nargs: '+',
203	type: maybeCreateReadStream,
204	help: 'Input datasets to evaluate (in TSV format); use - for standard input'
205	});
206	},
207
208	async execute(args) {
209	readAllLines(args.input_file)	×
210	.pipe(new DatasetParser({ contextual: args.contextual }))
211	.pipe(new Stream.Transform({
212	objectMode: true,
213
214	transform(ex, encoding, callback) {
215	const [newSentence, newProgram] = requoteSentence(ex.id, ex.preprocessed, ex.target_code);	×
216	ex.preprocessed = newSentence;	×
217	ex.target_code = newProgram;	×
218	callback(null, ex);	×
219	},
220
221	flush(callback) {
222	process.nextTick(callback);	×
223	}
224	}))
225	.pipe(new DatasetStringifier())
226	.pipe(args.output);
227
228	await StreamUtils.waitFinish(args.output);	×
229	}
230	};

stanford-oval / genie-toolkit / 591

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous