1336

pending completion

Build # 1336

Build Type

Pull #356

travis-ci-com

Committed by

web-flow

Commit Message

Merge c4caa6446 into 56e7ab2d7

Pull Request Pull Request #356: dispatcher: don't override an explicitly set NLU server URL

Run Details

6922 of 13035 branches covered (53.1%)

Branch coverage included in aggregate %.

2 of 4 new or added lines in 1 file covered. (50.0%)

12 existing lines in 5 files now uncovered.

15593 of 24862 relevant lines covered (62.72%)

1661225.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.34

/lib/dataset-tools/requoting.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

function findSubstring(sequence, substring) {
    for (let i = 0; i < sequence.length - substring.length + 1; i++) {
        let found = true;
        for (let j = 0; j < substring.length; j++) {
            if (sequence[i+j] !== substring[j]) {
                found = false;
                break;
            }
        }
        if (found)
            return i;
    }
    return -1;
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;

function findSpanType(program, begin_index, end_index) {
    let spanType;
    if (begin_index > 1 && program[begin_index-2] === 'location:') {
        spanType = 'LOCATION';
    } else if (end_index === program.length - 1 || !program[end_index+1].startsWith('^^')) {
        spanType = 'QUOTED_STRING';
    } else {
        switch (program[end_index+1]) {
        case '^^tt:hashtag':
            spanType = 'HASHTAG';
            break;
        case '^^tt:username':
            spanType = 'USERNAME';
            break;
        default:
            spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);
        }
        end_index++;
    }
    return [spanType, end_index];
}

function* requoteSentence(id, sentence, program) {
    sentence = sentence.split(' ');
    program = program.split(' ');

    const spans = [];
    let in_string = false;
    let begin_index = null;
    let end_index = null;
    for (let i = 0; i < program.length; i++) {
        let token = program[i];
        if (token === '"') {
            in_string = !in_string;
            if (in_string) {
                begin_index = i+1;
            } else {
                end_index = i;
                const substring = program.slice(begin_index, end_index);
                const idx = findSubstring(sentence, substring);
                if (idx < 0)
                    throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);

                const spanBegin = idx;
                const spanEnd = idx+end_index-begin_index;

                let spanType;
                [spanType, end_index] = findSpanType(program, begin_index, end_index);
                spans.push([spanBegin, spanEnd, spanType]);
                i = end_index;
            }
        }
    }

    if (spans.length === 0) {
        for (let i = 0; i < sentence.length; i++) {
            let word = sentence[i];
            // if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null)
                yield entityMatch[1];
            else
                yield word;
        }
        return;
    }

    spans.sort((a, b) => {
        const [abegin, aend] = a;
        const [bbegin, bend] = b;
        if (abegin < bbegin)
            return -1;
        if (bbegin < abegin)
            return +1;
        if (aend < bend)
            return -1;
        if (bend < aend)
            return +1;
        return 0;
    });
    let current_span_idx = 0;
    let current_span = spans[0];
    for (let i = 0; i < sentence.length; i++) {
        let word = sentence[i];
        if (current_span === null || i < current_span[0]) {
            // if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
            //
            // this means that the sentences
            // " tweet foo bar "
            // and
            // " tweet QUOTED_STRING_0 "
            //
            // will both turn into " tweet QUOTED_STRING "
            // and therefore match
            //
            // (this is only relevant if some portion of the data is left quoted
            // after augmentation)
            //
            // for simplicity, we do it only all entities
            // it has no effect on other entities because ids are assigned in increasing
            // number in the sentence
            const entityMatch = ENTITY_MATCH_REGEX.exec(word);
            if (entityMatch !== null)
                yield entityMatch[1];
            else
                yield word;
        } else if (i === current_span[0]) {
            yield current_span[2];
        } else if (i >= current_span[1]) {
            yield word;
            current_span_idx += 1;
            current_span = current_span_idx < spans.length ? spans[current_span_idx] : null;
        }
    }
}

function* requoteProgram(program) {
    if (typeof program === 'string')
        program = program.split(' ');

    let inString = false;
    let begin_index;
    for (let i = 0; i < program.length; i++) {
        const token = program[i];

        if (token === '"') {
            inString = !inString;
            if (inString) {
                begin_index = i+1;
            } else {
                const [spanType, end_index] = findSpanType(program, begin_index, i);
                yield spanType;
                i = end_index;
            }
        } else if (!inString) {
            const entityMatch = ENTITY_MATCH_REGEX.exec(token);
            if (entityMatch !== null)
                yield entityMatch[1];
            else if (token !== 'location:')
                yield token;
        }
    }
}

function* getFunctions(program, ignored = []) {
    if (typeof program === 'string')
        program = program.split(' ');
    let inString = false;
    let isDialoguePolicy = false;
    for (let token of program) {
        if (token === '"') {
            inString = !inString;
        } else if (!inString && token.startsWith('$dialogue')) {
            isDialoguePolicy = true;
        } else if (!inString && token.startsWith('@')) {
            if (isDialoguePolicy) {
                isDialoguePolicy = false;
                continue; // discard. It's the policy, not a function
            }
            if (ignored.includes(token))
                continue;
            yield token;
        }
    }
}

function* getDevices(program, ignored) {
    for (let fn of getFunctions(program, ignored)) {
        let dot = fn.lastIndexOf('.');
        yield fn.substring(0, dot);
    }
}

module.exports = {
    requoteSentence,
    requoteProgram,
    getFunctions,
    getDevices
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	function findSubstring(sequence, substring) {
23	for (let i = 0; i < sequence.length - substring.length + 1; i++) {	16,355✔
24	let found = true;	219,906✔
25	for (let j = 0; j < substring.length; j++) {	219,906✔
26	if (sequence[i+j] !== substring[j]) {	238,159✔
27	found = false;	203,551✔
28	break;	203,551✔
29	}
30	}
31	if (found)	219,906✔
32	return i;	16,355✔
33	}
34	return -1;	×
35	}
36
37	const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;	52✔
38
39	function findSpanType(program, begin_index, end_index) {
40	let spanType;
41	if (begin_index > 1 && program[begin_index-2] === 'location:') {	16,802✔
42	spanType = 'LOCATION';	228✔
43	} else if (end_index === program.length - 1 \|\| !program[end_index+1].startsWith('^^')) {	16,574✔
44	spanType = 'QUOTED_STRING';	16,050✔
45	} else {
46	switch (program[end_index+1]) {	524✔
47	case '^^tt:hashtag':
48	spanType = 'HASHTAG';	2✔
49	break;	2✔
50	case '^^tt:username':
51	spanType = 'USERNAME';	436✔
52	break;	436✔
53	default:
54	spanType = 'GENERIC_ENTITY_' + program[end_index+1].substring(2);	86✔
55	}
56	end_index++;	524✔
57	}
58	return [spanType, end_index];	16,802✔
59	}
60
61	function* requoteSentence(id, sentence, program) {
62	sentence = sentence.split(' ');	22,020✔
63	program = program.split(' ');	22,020✔
64
65	const spans = [];	22,020✔
66	let in_string = false;	22,020✔
67	let begin_index = null;	22,020✔
68	let end_index = null;	22,020✔
69	for (let i = 0; i < program.length; i++) {	22,020✔
70	let token = program[i];	440,925✔
71	if (token === '"') {	440,925✔
72	in_string = !in_string;	32,710✔
73	if (in_string) {	32,710✔
74	begin_index = i+1;	16,355✔
75	} else {
76	end_index = i;	16,355✔
77	const substring = program.slice(begin_index, end_index);	16,355✔
78	const idx = findSubstring(sentence, substring);	16,355✔
79	if (idx < 0)	16,355!
80	throw new Error(`Cannot find span ${substring.join(' ')} in sentence id ${id}`);	×
81
82	const spanBegin = idx;	16,355✔
83	const spanEnd = idx+end_index-begin_index;	16,355✔
84
85	let spanType;
86	[spanType, end_index] = findSpanType(program, begin_index, end_index);	16,355✔
87	spans.push([spanBegin, spanEnd, spanType]);	16,355✔
88	i = end_index;	16,355✔
89	}
90	}
91	}
92
93	if (spans.length === 0) {	22,020✔
94	for (let i = 0; i < sentence.length; i++) {	9,737✔
95	let word = sentence[i];	142,938✔
96	// if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
97	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	142,938✔
98	if (entityMatch !== null)	142,938✔
99	yield entityMatch[1];	1,994✔
100	else
101	yield word;	140,944✔
102	}
103	return;	9,737✔
104	}
105
106	spans.sort((a, b) => {	12,283✔
107	const [abegin, aend] = a;	4,152✔
108	const [bbegin, bend] = b;	4,152✔
109	if (abegin < bbegin)	4,152✔
110	return -1;	1,726✔
111	if (bbegin < abegin)	2,426✔
112	return +1;	2,116✔
113	if (aend < bend)	310✔
114	return -1;	1✔
115	if (bend < aend)	309!
UNCOV 116	return +1;	×
117	return 0;	309✔
118	});
119	let current_span_idx = 0;	12,283✔
120	let current_span = spans[0];	12,283✔
121	for (let i = 0; i < sentence.length; i++) {	12,283✔
122	let word = sentence[i];	225,772✔
123	if (current_span === null \|\| i < current_span[0]) {	225,772✔
124	// if the current word is a QUOTED_STRING, HASHTAG, ENTITY or GENERIC_ENTITY_, remove the last _<id> part
125	//
126	// this means that the sentences
127	// " tweet foo bar "
128	// and
129	// " tweet QUOTED_STRING_0 "
130	//
131	// will both turn into " tweet QUOTED_STRING "
132	// and therefore match
133	//
134	// (this is only relevant if some portion of the data is left quoted
135	// after augmentation)
136	//
137	// for simplicity, we do it only all entities
138	// it has no effect on other entities because ids are assigned in increasing
139	// number in the sentence
140	const entityMatch = ENTITY_MATCH_REGEX.exec(word);	178,795✔
141	if (entityMatch !== null)	178,795✔
142	yield entityMatch[1];	5,526✔
143	else
144	yield word;	173,269✔
145	} else if (i === current_span[0]) {	46,977✔
146	yield current_span[2];	16,042✔
147	} else if (i >= current_span[1]) {	30,935✔
148	yield word;	13,105✔
149	current_span_idx += 1;	13,105✔
150	current_span = current_span_idx < spans.length ? spans[current_span_idx] : null;	13,105✔
151	}
152	}
153	}
154
155	function* requoteProgram(program) {
156	if (typeof program === 'string')	580!
157	program = program.split(' ');	580✔
158
159	let inString = false;	580✔
160	let begin_index;
161	for (let i = 0; i < program.length; i++) {	580✔
162	const token = program[i];	11,158✔
163
164	if (token === '"') {	11,158✔
165	inString = !inString;	894✔
166	if (inString) {	894✔
167	begin_index = i+1;	447✔
168	} else {
169	const [spanType, end_index] = findSpanType(program, begin_index, i);	447✔
170	yield spanType;	447✔
171	i = end_index;	447✔
172	}
173	} else if (!inString) {	10,264✔
174	const entityMatch = ENTITY_MATCH_REGEX.exec(token);	9,028✔
175	if (entityMatch !== null)	9,028✔
176	yield entityMatch[1];	111✔
177	else if (token !== 'location:')	8,917✔
178	yield token;	8,910✔
179	}
180	}
181	}
182
183	function* getFunctions(program, ignored = []) {	576✔
184	if (typeof program === 'string')	19,096!
185	program = program.split(' ');	19,096✔
186	let inString = false;	19,096✔
187	let isDialoguePolicy = false;	19,096✔
188	for (let token of program) {	19,096✔
189	if (token === '"') {	335,109✔
190	inString = !inString;	892✔
191	} else if (!inString && token.startsWith('$dialogue')) {	334,217✔
192	isDialoguePolicy = true;	2,354✔
193	} else if (!inString && token.startsWith('@')) {	331,863✔
194	if (isDialoguePolicy) {	24,198✔
195	isDialoguePolicy = false;	2,354✔
196	continue; // discard. It's the policy, not a function	2,354✔
197	}
198	if (ignored.includes(token))	21,844✔
199	continue;	627✔
200	yield token;	21,217✔
201	}
202	}
203	}
204
205	function* getDevices(program, ignored) {
206	for (let fn of getFunctions(program, ignored)) {	18,526✔
207	let dot = fn.lastIndexOf('.');	20,111✔
208	yield fn.substring(0, dot);	20,111✔
209	}
210	}
211
212	module.exports = {	52✔
213	requoteSentence,
214	requoteProgram,
215	getFunctions,
216	getDevices
217	};

stanford-oval / genie-toolkit / 1336

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous