stanford-oval / genie-toolkit / 717

pending completion

Build # 717

Build Type

Pull #145

travis-ci-com

Committed by

web-flow

Commit Message

Merge cef1e52eb into f2e74d1c4

Pull Request Pull Request #145: Inference mode dialogue templates

Run Details

1278 of 4818 branches covered (26.53%)

Branch coverage included in aggregate %.

75 of 314 new or added lines in 11 files covered. (23.89%)

5 existing lines in 3 files now uncovered.

4029 of 9770 relevant lines covered (41.24%)

7848.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.36

/lib/requoting.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2018-2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

function findSubstring(sequence, substring) {
    for (let i = 0; i < sequence.length - substring.length + 1; i++) {
        let found = true;
        for (let j = 0; j < substring.length; j++) {
            if (sequence[i+j] !== substring[j]) {
                found = false;
                break;
            }
        }
        if (found)
            return i;
    }
    return -1;
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;

function findSpanType(program, begin_index, end_index) {
    let spanType;
    if (begin_index > 1 && program[begin_index-2] === 'location:') {
        spanType = 'LOCATION';
    } else if (end_index === program.length - 1 || !program[end_index+1].startsWith('^^')) {
        spanType = 'QUOTED_STRING';
    } else {
        switch (program[end_index+1]) {
        case '^^tt:hashtag':
            spanType = 'HASHTAG';
            break;
        case '^^tt:username':
            spanType = 'USERNAME';
            break;
        default:
            spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);
        }
        end_index++;
    }
    return [spanType, end_index];
}

function* requoteSentence(id, sentence, program) {
    sentence = sentence.split(' ');
    program = program.split(' ');

    const spans = [];
    let in_string = false;
    let begin_index = null;
    let end_index = null;
    for (let i = 0; i < program.length; i++) {
        let token = program[i];
        if (token === '"') {
            in_string = !in_string;
            if (in_string) {
                begin_index = i+1;
            } else {
                end_index = i;
                const substring = program.slice(begin_index, end_index);
                const idx = findSubstring(sentence, substring);
                if (idx < 0)
                    throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);

                const spanBegin = idx;
                const spanEnd = idx+end_index-begin_index;

                let spanType;
                [spanType, end_index] = findSpanType(program, begin_index, end_index);
                spans.push([spanBegin, spanEnd, spanType]);
                i = end_index;
            }
        }
    }

    if (spans.length === 0) {
        for (let i = 0; i < sentence.length; i++) {
            let word = sentence[i];
            // if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null)
                yield entityMatch[1];
            else
                yield word;
        }
        return;
    }

    spans.sort((a, b) => {
        const [abegin, aend] = a;
        const [bbegin, bend] = b;
        if (abegin < bbegin)
            return -1;
        if (bbegin < abegin)
            return +1;
        if (aend < bend)
            return -1;
        if (bend < aend)
            return +1;
        return 0;
    });
    let current_span_idx = 0;
    let current_span = spans[0];
    for (let i = 0; i < sentence.length; i++) {
        let word = sentence[i];
        if (current_span === null || i < current_span[0]) {
            // if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
            //
            // this means that the sentences
            // " tweet foo bar "
            // and
            // " tweet QUOTED_STRING_0 "
            //
            // will both turn into " tweet QUOTED_STRING "
            // and therefore match
            //
            // (this is only relevant if some portion of the data is left quoted
            // after augmentation)
            //
            // for simplicity, we do it only all entities
            // it has no effect on other entities because ids are assigned in increasing
            // number in the sentence
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null)
                yield entityMatch[1];
            else
                yield word;
        } else if (i === current_span[0]) {
            yield current_span[2];
        } else if (i >= current_span[1]) {
            yield word;
            current_span_idx += 1;
            current_span = current_span_idx < spans.length ? spans[current_span_idx] : null;
        }
    }
}

function* requoteProgram(program) {
    if (typeof program === 'string')
        program = program.split(' ');

    let inString = false;
    let begin_index;
    for (let i = 0; i < program.length; i++) {
        const token = program[i];

        if (token === '"') {
            inString = !inString;
            if (inString) {
                begin_index = i+1;
            } else {
                const [spanType, end_index] = findSpanType(program, begin_index, i);
                yield spanType;
                i = end_index;
            }
        } else if (!inString) {
            const entityMatch = ENTITY_MATCH_REGEX.exec(token);
            if (entityMatch !== null)
                yield entityMatch[1];
            else if (token !== 'location:')
                yield token;
        }
    }
}

function* getFunctions(program) {
    if (typeof program === 'string')
        program = program.split(' ');
    let inString = false;
    for (let token of program) {
        if (token === '"')
            inString = !inString;
        else if (!inString && token.startsWith('@'))
            yield token;
    }
}

function* getDevices(program) {
    for (let fn of getFunctions(program)) {
        let dot = fn.lastIndexOf('.');
        yield fn.substring(0, dot);
    }
}

module.exports = {
    requoteSentence,
    requoteProgram,
    getFunctions,
    getDevices
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2018-2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	function findSubstring(sequence, substring) {
13	for (let i = 0; i < sequence.length - substring.length + 1; i++) {	845✔
14	let found = true;	10,221✔
15	for (let j = 0; j < substring.length; j++) {	10,221✔
16	if (sequence[i+j] !== substring[j]) {	11,883✔
17	found = false;	9,376✔
18	break;	9,376✔
19	}
20	}
21	if (found)	10,221✔
22	return i;	845✔
23	}
24	return -1;	×
25	}
26
27	const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;	24✔
28
29	function findSpanType(program, begin_index, end_index) {
30	let spanType;
31	if (begin_index > 1 && program[begin_index-2] === 'location:') {	1,325✔
32	spanType = 'LOCATION';	39✔
33	} else if (end_index === program.length - 1 \|\| !program[end_index+1].startsWith('^^')) {	1,286✔
34	spanType = 'QUOTED_STRING';	1,244✔
35	} else {
36	switch (program[end_index+1]) {	42✔
37	case '^^tt:hashtag':
38	spanType = 'HASHTAG';	2✔
39	break;	2✔
40	case '^^tt:username':
41	spanType = 'USERNAME';	38✔
42	break;	38✔
43	default:
44	spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);	2✔
45	}
46	end_index++;	42✔
47	}
48	return [spanType, end_index];	1,325✔
49	}
50
51	function* requoteSentence(id, sentence, program) {
52	sentence = sentence.split(' ');	1,038✔
53	program = program.split(' ');	1,038✔
54
55	const spans = [];	1,038✔
56	let in_string = false;	1,038✔
57	let begin_index = null;	1,038✔
58	let end_index = null;	1,038✔
59	for (let i = 0; i < program.length; i++) {	1,038✔
60	let token = program[i];	20,241✔
61	if (token === '"') {	20,241✔
62	in_string = !in_string;	1,690✔
63	if (in_string) {	1,690✔
64	begin_index = i+1;	845✔
65	} else {
66	end_index = i;	845✔
67	const substring = program.slice(begin_index, end_index);	845✔
68	const idx = findSubstring(sentence, substring);	845✔
69	if (idx < 0)	845!
70	throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);	×
71
72	const spanBegin = idx;	845✔
73	const spanEnd = idx+end_index-begin_index;	845✔
74
75	let spanType;
76	[spanType, end_index] = findSpanType(program, begin_index, end_index);	845✔
77	spans.push([spanBegin, spanEnd, spanType]);	845✔
78	i = end_index;	845✔
79	}
80	}
81	}
82
83	if (spans.length === 0) {	1,038✔
84	for (let i = 0; i < sentence.length; i++) {	194✔
85	let word = sentence[i];	2,380✔
86	// if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
87	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	2,380✔
88	if (entityMatch !== null)	2,380✔
89	yield entityMatch[1];	30✔
90	else
91	yield word;	2,350✔
92	}
93	return;	194✔
94	}
95
96	spans.sort((a, b) => {	844✔
97	const [abegin, aend] = a;	1✔
98	const [bbegin, bend] = b;	1✔
99	if (abegin < bbegin)	1!
100	return -1;	1✔
UNCOV 101	if (bbegin < abegin)	×
UNCOV 102	return +1;	×
103	if (aend < bend)	×
104	return -1;	×
105	if (bend < aend)	×
106	return +1;	×
107	return 0;	×
108	});
109	let current_span_idx = 0;	844✔
110	let current_span = spans[0];	844✔
111	for (let i = 0; i < sentence.length; i++) {	844✔
112	let word = sentence[i];	14,890✔
113	if (current_span === null \|\| i < current_span[0]) {	14,890✔
114	// if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
115	//
116	// this means that the sentences
117	// " tweet foo bar "
118	// and
119	// " tweet QUOTED_STRING_0 "
120	//
121	// will both turn into " tweet QUOTED_STRING "
122	// and therefore match
123	//
124	// (this is only relevant if some portion of the data is left quoted
125	// after augmentation)
126	//
127	// for simplicity, we do it only all entities
128	// it has no effect on other entities because ids are assigned in increasing
129	// number in the sentence
130	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	11,832✔
131	if (entityMatch !== null)	11,832✔
132	yield entityMatch[1];	64✔
133	else
134	yield word;	11,768✔
135	} else if (i === current_span[0]) {	3,058✔
136	yield current_span[2];	845✔
137	} else if (i >= current_span[1]) {	2,213✔
138	yield word;	631✔
139	current_span_idx += 1;	631✔
140	current_span = current_span_idx < spans.length ? spans[current_span_idx] : null;	631✔
141	}
142	}
143	}
144
145	function* requoteProgram(program) {
146	if (typeof program === 'string')	677!
147	program = program.split(' ');	677✔
148
149	let inString = false;	677✔
150	let begin_index;
151	for (let i = 0; i < program.length; i++) {	677✔
152	const token = program[i];	12,289✔
153
154	if (token === '"') {	12,289✔
155	inString = !inString;	960✔
156	if (inString) {	960✔
157	begin_index = i+1;	480✔
158	} else {
159	const [spanType, end_index] = findSpanType(program, begin_index, i);	480✔
160	yield spanType;	480✔
161	i = end_index;	480✔
162	}
163	} else if (!inString) {	11,329✔
164	const entityMatch = ENTITY_MATCH_REGEX.exec(token);	9,995✔
165	if (entityMatch !== null)	9,995✔
166	yield entityMatch[1];	94✔
167	else if (token !== 'location:')	9,901✔
168	yield token;	9,882✔
169	}
170	}
171	}
172
173	function* getFunctions(program) {
174	if (typeof program === 'string')	664!
175	program = program.split(' ');	664✔
176	let inString = false;	664✔
177	for (let token of program) {	664✔
178	if (token === '"')	12,258✔
179	inString = !inString;	948✔
180	else if (!inString && token.startsWith('@'))	11,310✔
181	yield token;	1,238✔
182	}
183	}
184
185	function* getDevices(program) {
186	for (let fn of getFunctions(program)) {	×
187	let dot = fn.lastIndexOf('.');	×
188	yield fn.substring(0, dot);	×
189	}
190	}
191
192	module.exports = {	24✔
193	requoteSentence,
194	requoteProgram,
195	getFunctions,
196	getDevices
197	};

stanford-oval / genie-toolkit / 717

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous