stanford-oval / genie-toolkit / 735

pending completion

Build # 735

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Merge pull request #152 from stanford-oval/wip/multilanguage

Wip/multilanguage

Run Details

1349 of 4759 branches covered (28.35%)

Branch coverage included in aggregate %.

81 of 102 new or added lines in 6 files covered. (79.41%)

3 existing lines in 1 file now uncovered.

4075 of 9606 relevant lines covered (42.42%)

17363.44 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.08

/lib/i18n/american-english.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2018-2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');
const { coin } = require('../random');
const Inflectors = require('en-inflectors').Inflectors;
const Tag = require('en-pos').Tag;

function replaceMeMy(sentence) {
    sentence = sentence.replace(/\b((?!(?:let|inform|notify|alert|send)\b)[a-zA-Z0-9]+) me\b/g, '$1 them');

    return sentence.replace(/\b(my|i|mine)\b/g, (what) => {
        switch(what) {
        case 'me':
            return 'them';
        case 'my':
            return 'their';
        case 'mine':
            return 'theirs';
        case 'i':
            return 'they';
        default:
            return what;
        }
    });
}

// Apply final touches to a newly generated synthetic sentence
//
// This function should correct coreferences, conjugations and other
// grammar/readability issues that are too inconvenient to prevent
// using the templates 
function postprocessSynthetic(sentence, program, rng) {
    assert(rng);
    if (program.isProgram && program.principal !== null)
        sentence = replaceMeMy(sentence);

    if (!sentence.endsWith(' ?') && !sentence.endsWith(' !') && !sentence.endsWith(' .') && coin(0.5, rng))
        sentence = sentence.trim() + ' .';
    if (sentence.endsWith(' ?') && coin(0.5, rng))
        sentence = sentence.substring(0, sentence.length-2);

    sentence = sentence.replace(/ (1|one|a) ([a-z]+)s /g, ' $1 $2 ');

    if (coin(0.5, rng))
        sentence = sentence.replace(/ with (no|zero) /g, ' without ');

    if (coin(0.5, rng))
        sentence = sentence.replace(/ has no /g, ' does not have ');
    if (coin(0.5, rng))
        sentence = sentence.replace(/ have no /g, ' do not have ');

    // contractions
    // FIXME check multiwoz preprocessing is compatible
    // FIXME we should probably either always do them or always not do them for
    // agent utterances, otherwise the agent will speak weirdly
    if (coin(0.5, rng))
        sentence = sentence.replace(/ (does|do) not /g, ' $1 n\'t ');
    if (coin(0.5, rng))
        sentence = sentence.replace(/ (he|she|it|what|who|where|when) (is|has) /g, ' $1 \'s ');
    if (coin(0.5, rng))
        sentence = sentence.replace(/ i am /g, ' i \'m ');
    if (coin(0.5, rng))
        sentence = sentence.replace(/ (you|we|they) are /g, ' $1 \'re ');
    if (coin(0.5, rng))
        sentence = sentence.replace(/ (i|you|he|she|we|they) (had|would) /g, ' $1 \'d ');

    sentence = sentence.replace(/#([a-z_]*) ((?:a|an|the) )?(NUMBER_[0-9]|0|zero|1|one|QUOTED_STRING_[0-9]|GENERIC_ENTITY_[^ ]+_[0-9])/g,
        (_, word, mid, value) => (mid || '') + value + ' ' + word.replace(/_/g, ' '));

    sentence = sentence.replace(/ (a|the) something /g, ' something ');

    //sentence = sentence.replace(/ a ([a-z]+) -s /g, ' $1 -s ');

    sentence = sentence.replace(/ ([a-z]+) -ly /g, ' $1ly ');

    sentence = sentence.replace(/ a (?!one )(?=[aeiou])/g, ' an ');

    sentence = sentence.replace(/ new (their|my|the|a) /, (_, what) => ` ${what} new `);

    sentence = sentence.replace(/ 's (my|their|his|her) /, ` 's `); //'

    // remove extra # introduced by annotations, and not yet been replaced by value
    sentence = sentence.replace(/#/g, '');

    return sentence.trim();
}

// Override the canonical form of argument names for synthetic generation
// (to generate filters and projections)
//
// More than one form can be provided for each argument name, in which case
// all are used
//
// FIXME this info should be in Thingpedia
// if there is only a single value, this is possible without changing the parameter
// name by adding a #_[canonical] annotation
const ARGUMENT_NAME_OVERRIDES = {
    'updated': ['update time'],
    'random': ['random number'],

    'picture_url': ['picture', 'image', 'photo'],

    'title': ['headline', 'title'],

    'file_name': ['file name', 'name'],
    'file_size': ['file size', 'size', 'disk usage'],
    // not even silei knows about mime types, so definitely no mime type here!
    'mime_type': ['file type', 'type'],

    'user_name': ['username', 'user name'],
};

// Tokens that can be ignored in the names of entities, by entity type
//
// This should cover abbreviations, prefixes and suffixes that are usually
// omitted in colloquial speech
const IGNORABLE_TOKENS = {
    'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
    'imgflip:meme_id': ['the'],
    'tt:currency_code': ['us'],
    'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
    'org:freedesktop:app_id': ['gnome']
};
// Interchangeable abbreviations for entity names
//
// Each entry in this array is a set (in array form) of abbreviations with the same
// meaning; while expanding parameters, one of the possible forms is chosen at random
//
// Use this to fix tokenization inconsistencies in the entity database, to add
// colloquial forms, and to add robustness to punctuation
const ABBREVIATIONS = [
    ['ltd', 'ltd.', 'limited'],
    ['corp', 'corp.', 'corporation'],
    ['l.l.c', 'llc'],
    ['&', 'and'],
    ['inc.', 'inc', 'incorporated'],
];
const PROCESSED_ABBREVIATIONS = {};
for (let abbr of ABBREVIATIONS) {
    for (let variant of abbr)
        PROCESSED_ABBREVIATIONS[variant] = abbr;

}

// tokens that are treated specially by the PTB tokenizer for English
// (and what they map to in properly punctuated English)
const SPECIAL_TOKENS = {
    '.': '.',
    ',': ',',
    'n\'t': 'n\'t',
    '\'s': '\'s',
    '?': '?',

    // right/left round/curly/square bracket
    '-rrb-': ')',
    '-lrb-': ' (',
    '-rcb-': '}',
    '-lcb-': ' {',
    '-rsb-': ']',
    '-lsb-': ' [',
};


// Convert a tokenized sentence back into a correctly spaced, correctly punctuated
// sentence, to present to an MTurk worker for paraphrasing
function detokenize(sentence, prevtoken, token) {
    if (token in SPECIAL_TOKENS) {
        sentence += SPECIAL_TOKENS[token];
    } else if ((token === 'not' && prevtoken === 'can') ||
        ((token === 'na' || token === 'ta') && prevtoken === 'gon')) {
        // PTB tokenizer does the following
        // cannot -> can not
        // gonna -> gon na
        // gotta -> got ta
        // invert it here
        //
        // note the absence of a space
        sentence += token;
    } else {
        if (sentence)
            sentence += ' ';
        sentence += token;
    }
    return sentence;
}

function detokenizeSentence(tokens) {
    let sentence = '';
    let prevToken = '';
    for (let token of tokens) {
        sentence = detokenize(sentence, prevToken, token);
        prevToken = token;
    }
    return sentence;
}

function capitalize(word) {
    return word[0].toUpperCase() + word.substring(1);
}

const MUST_CAPITALIZE_TOKEN = new Set([
    'i',

    'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
    'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september',
    'october', 'november', 'december',

    // HACK
    'chinese', 'italian', 'french', 'english', 'american'
]);

function postprocessNLG(answer, entities) {
    // simple true-casing: uppercase all letters at the beginning of the sentence
    // and after a period, question or exclamation mark
    answer = answer.replace(/(^| [.?!] )([a-z])/g, (_, prefix, letter) => prefix + letter.toUpperCase());

    answer = answer.split(' ').map((token) => {
        if (token in entities) {
            if (token.startsWith('GENERIC_ENTITY_'))
                return (entities[token].display || entities[token].value);
            return String(entities[token]);
        }

        // capitalize certain tokens that should be capitalized in English
        if (MUST_CAPITALIZE_TOKEN.has(token))
            return capitalize(token);
        return token;
    });
    answer = detokenizeSentence(answer);

    return answer;
}

// All the different forms in which MTurk workers write "no idea" for a sentence
// they don't understand
//
// This is usually empirically collected by looking at the results and finding
// sentences that don't validate or are too short
const NO_IDEA = [
    'no idea', 'don\'t know', 'dont know', 'don\'t understand',
    'dont understand', 'no clue',
    'doesn\'t make sense', 'doesn\'t make any sense',
    'doesnt make sense', 'doesnt make any sense'
];

// tb is terabyte in our dataset, tubercolosis in PPDB
// channel is TV/YouTube channel for us, river/waterway for PPDB
const PPDB_BLACKLIST = new Set(['tb', 'channel']);

// Check if a pair of word, paraphrase from PPDB should be considered a candidate
// for augmentation or not
function isValidParaphrasePair(word, paraphrase) {
    if (PPDB_BLACKLIST.has(word))
        return false;
    // ignore singular/plural relation and verb/gerund
    if (paraphrase === word + 's' || word === paraphrase + 's')
        return false;
    if (paraphrase === word + 'ing' || word === paraphrase + 'ing')
        return false;

    // don't change the mode or tense of the verb
    if (paraphrase.endsWith('ing') !== word.endsWith('ing'))
        return false;
    if (paraphrase.endsWith('ed') !== word.endsWith('ed'))
        return false;
    return true;
}

const CHANGE_SUBJECT_TEMPLATES = [
    'ok , how about {}',
    'how about {} instead',
    'no {}',
    'no i said {}',
    'no , i said {}',
    'i said {}',
    'i want {} instead',
    'no instead {}',
];

const SINGLE_DEVICE_TEMPLATES = [
    ['ask $device to $command', /^(?!(?:what|when|how|who) )/],
    ['ask $device about $command', /^(?:what|when|how|who) /],
    ['ask $device for $command', /^the /],
    ['ask $device $command', /^(?:what|when|how|who) /],
    ['tell $device to $command', /^(?!(?:what|when|how|who) )/],
    ['tell $device that $command', /^(?!(?:what|when|how|who) )/],
    ['use $device to $command', /^(?!(?:what|when|how|who) )/],
    ['use $device and $command', /^(?!(?:what|when|how|who) )/],
    ['order $device to $command', /^(?!(?:what|when|how|who) )/],
    ['$command from $device', null],
    ['$command using $device', null],
    ['$command in $device', null],
    ['$command by $device', null],
    ['$command with $device', null],
    ['talk to $device and $command', /^(?!(?:what|when|how|who) )/],
    ['open $device and $command', /^(?!(?:what|when|how|who) )/],
    ['launch $device and $command', /^(?!(?:what|when|how|who) )/],
];

function pluralize(name) {
    if (!name.includes(' ')) {
        if (new Tag([name]).initial().tags[0] === 'NN')
            return new Inflectors(name).toPlural();
        return name;
    } else {
        const words = name.split(' ');
        const tags = new Tag(words).initial().tags;
        if (tags[tags.length - 1] !== 'NN')
            return name;
        else if (['VB', 'VBP', 'VBZ', 'VBD'].includes(tags[0]))
            return name;
        words[words.length - 1] = pluralize(words[words.length - 1]);
        return words.join(' ');
    }
}

function isGoodWord(word) {
    // filter out words that cannot be in the dataset,
    // because they would be either tokenized/preprocessed out or
    // they are unlikely to be used with voice
    return /^([a-zA-Z0-9-][a-zA-Z0-9.-]*|'s|,|\?)$/.test(word);
}

function isGoodSentence(sentence) {
    if (sentence.length < 3)
        return false;
    if (['.', ',', '?', '!', ' '].includes(sentence[0]))
        return false;
    return !/^(the|a|has|for|title|when|me|i|so|--|to|up|and|or|of|is|are|be|being|have|an|that|this|these|those|with|as|in|on|before|after)$/.test(sentence);
}

function isGoodNumber(number) {
    return /^([0-9]+)$/.test(number);
}


module.exports = {
    postprocessSynthetic,
    detokenize,
    postprocessNLG,

    pluralize,

    ARGUMENT_NAME_OVERRIDES,

    IGNORABLE_TOKENS,
    ABBREVIATIONS: PROCESSED_ABBREVIATIONS,

    NO_IDEA,

    CHANGE_SUBJECT_TEMPLATES,
    SINGLE_DEVICE_TEMPLATES,

    isValidParaphrasePair,

    isGoodWord,
    isGoodSentence,
    isGoodNumber
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2018-2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	52✔
13	const { coin } = require('../random');	52✔
14	const Inflectors = require('en-inflectors').Inflectors;	52✔
15	const Tag = require('en-pos').Tag;	52✔
16
17	function replaceMeMy(sentence) {
18	sentence = sentence.replace(/\b((?!(?:let\|inform\|notify\|alert\|send)\b)[a-zA-Z0-9]+) me\b/g, '$1 them');	×
19
20	return sentence.replace(/\b(my\|i\|mine)\b/g, (what) => {	×
21	switch(what) {	×
22	case 'me':
23	return 'them';	×
24	case 'my':
25	return 'their';	×
26	case 'mine':
27	return 'theirs';	×
28	case 'i':
29	return 'they';	×
30	default:
31	return what;	×
32	}
33	});
34	}
35
36	// Apply final touches to a newly generated synthetic sentence
37	//
38	// This function should correct coreferences, conjugations and other
39	// grammar/readability issues that are too inconvenient to prevent
40	// using the templates
41	function postprocessSynthetic(sentence, program, rng) {
42	assert(rng);	1,200,000✔
43	if (program.isProgram && program.principal !== null)	1,200,000!
44	sentence = replaceMeMy(sentence);	×
45
46	if (!sentence.endsWith(' ?') && !sentence.endsWith(' !') && !sentence.endsWith(' .') && coin(0.5, rng))	1,200,000✔
47	sentence = sentence.trim() + ' .';	241,786✔
48	if (sentence.endsWith(' ?') && coin(0.5, rng))	1,200,000✔
49	sentence = sentence.substring(0, sentence.length-2);	358,258✔
50
51	sentence = sentence.replace(/ (1\|one\|a) ([a-z]+)s /g, ' $1 $2 ');	1,200,000✔
52
53	if (coin(0.5, rng))	1,200,000✔
54	sentence = sentence.replace(/ with (no\|zero) /g, ' without ');	600,472✔
55
56	if (coin(0.5, rng))	1,200,000✔
57	sentence = sentence.replace(/ has no /g, ' does not have ');	599,756✔
58	if (coin(0.5, rng))	1,200,000✔
59	sentence = sentence.replace(/ have no /g, ' do not have ');	599,780✔
60
61	// contractions
62	// FIXME check multiwoz preprocessing is compatible
63	// FIXME we should probably either always do them or always not do them for
64	// agent utterances, otherwise the agent will speak weirdly
65	if (coin(0.5, rng))	1,200,000✔
66	sentence = sentence.replace(/ (does\|do) not /g, ' $1 n\'t ');	600,038✔
67	if (coin(0.5, rng))	1,200,000✔
68	sentence = sentence.replace(/ (he\|she\|it\|what\|who\|where\|when) (is\|has) /g, ' $1 \'s ');	599,912✔
69	if (coin(0.5, rng))	1,200,000✔
70	sentence = sentence.replace(/ i am /g, ' i \'m ');	600,634✔
71	if (coin(0.5, rng))	1,200,000✔
72	sentence = sentence.replace(/ (you\|we\|they) are /g, ' $1 \'re ');	601,062✔
73	if (coin(0.5, rng))	1,200,000✔
74	sentence = sentence.replace(/ (i\|you\|he\|she\|we\|they) (had\|would) /g, ' $1 \'d ');	601,068✔
75
76	sentence = sentence.replace(/#([a-z_]*) ((?:a\|an\|the) )?(NUMBER_[0-9]\|0\|zero\|1\|one\|QUOTED_STRING_[0-9]\|GENERIC_ENTITY_[^ ]+_[0-9])/g,	1,200,000✔
77	(_, word, mid, value) => (mid \|\| '') + value + ' ' + word.replace(/_/g, ' '));	×
78
79	sentence = sentence.replace(/ (a\|the) something /g, ' something ');	1,200,000✔
80
81	//sentence = sentence.replace(/ a ([a-z]+) -s /g, ' $1 -s ');
82
83	sentence = sentence.replace(/ ([a-z]+) -ly /g, ' $1ly ');	1,200,000✔
84
85	sentence = sentence.replace(/ a (?!one )(?=[aeiou])/g, ' an ');	1,200,000✔
86
87	sentence = sentence.replace(/ new (their\|my\|the\|a) /, (_, what) => ` ${what} new `);	1,200,000✔
88
89	sentence = sentence.replace(/ 's (my\|their\|his\|her) /, ` 's `); //'	1,200,000✔
90
91	// remove extra # introduced by annotations, and not yet been replaced by value
92	sentence = sentence.replace(/#/g, '');	1,200,000✔
93
94	return sentence.trim();	1,200,000✔
95	}
96
97	// Override the canonical form of argument names for synthetic generation
98	// (to generate filters and projections)
99	//
100	// More than one form can be provided for each argument name, in which case
101	// all are used
102	//
103	// FIXME this info should be in Thingpedia
104	// if there is only a single value, this is possible without changing the parameter
105	// name by adding a #_[canonical] annotation
106	const ARGUMENT_NAME_OVERRIDES = {	52✔
107	'updated': ['update time'],
108	'random': ['random number'],
109
110	'picture_url': ['picture', 'image', 'photo'],
111
112	'title': ['headline', 'title'],
113
114	'file_name': ['file name', 'name'],
115	'file_size': ['file size', 'size', 'disk usage'],
116	// not even silei knows about mime types, so definitely no mime type here!
117	'mime_type': ['file type', 'type'],
118
119	'user_name': ['username', 'user name'],
120	};
121
122	// Tokens that can be ignored in the names of entities, by entity type
123	//
124	// This should cover abbreviations, prefixes and suffixes that are usually
125	// omitted in colloquial speech
126	const IGNORABLE_TOKENS = {	52✔
127	'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
128	'imgflip:meme_id': ['the'],
129	'tt:currency_code': ['us'],
130	'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
131	'org:freedesktop:app_id': ['gnome']
132	};
133	// Interchangeable abbreviations for entity names
134	//
135	// Each entry in this array is a set (in array form) of abbreviations with the same
136	// meaning; while expanding parameters, one of the possible forms is chosen at random
137	//
138	// Use this to fix tokenization inconsistencies in the entity database, to add
139	// colloquial forms, and to add robustness to punctuation
140	const ABBREVIATIONS = [	52✔
141	['ltd', 'ltd.', 'limited'],
142	['corp', 'corp.', 'corporation'],
143	['l.l.c', 'llc'],
144	['&', 'and'],
145	['inc.', 'inc', 'incorporated'],
146	];
147	const PROCESSED_ABBREVIATIONS = {};	52✔
148	for (let abbr of ABBREVIATIONS) {	52✔
149	for (let variant of abbr)	260✔
150	PROCESSED_ABBREVIATIONS[variant] = abbr;	676✔
151
152	}
153
154	// tokens that are treated specially by the PTB tokenizer for English
155	// (and what they map to in properly punctuated English)
156	const SPECIAL_TOKENS = {	52✔
157	'.': '.',
158	',': ',',
159	'n\'t': 'n\'t',
160	'\'s': '\'s',
161	'?': '?',
162
163	// right/left round/curly/square bracket
164	'-rrb-': ')',
165	'-lrb-': ' (',
166	'-rcb-': '}',
167	'-lcb-': ' {',
168	'-rsb-': ']',
169	'-lsb-': ' [',
170	};
171
172
173	// Convert a tokenized sentence back into a correctly spaced, correctly punctuated
174	// sentence, to present to an MTurk worker for paraphrasing
175	function detokenize(sentence, prevtoken, token) {
176	if (token in SPECIAL_TOKENS) {	344✔
177	sentence += SPECIAL_TOKENS[token];	2✔
178	} else if ((token === 'not' && prevtoken === 'can') \|\|	342!
179	((token === 'na' \|\| token === 'ta') && prevtoken === 'gon')) {
180	// PTB tokenizer does the following
181	// cannot -> can not
182	// gonna -> gon na
183	// gotta -> got ta
184	// invert it here
185	//
186	// note the absence of a space
187	sentence += token;	×
188	} else {
189	if (sentence)	342✔
190	sentence += ' ';	308✔
191	sentence += token;	342✔
192	}
193	return sentence;	344✔
194	}
195
196	function detokenizeSentence(tokens) {
197	let sentence = '';	×
198	let prevToken = '';	×
199	for (let token of tokens) {	×
200	sentence = detokenize(sentence, prevToken, token);	×
201	prevToken = token;	×
202	}
203	return sentence;	×
204	}
205
206	function capitalize(word) {
207	return word[0].toUpperCase() + word.substring(1);	×
208	}
209
210	const MUST_CAPITALIZE_TOKEN = new Set([	52✔
211	'i',
212
213	'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
214	'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september',
215	'october', 'november', 'december',
216
217	// HACK
218	'chinese', 'italian', 'french', 'english', 'american'
219	]);
220
221	function postprocessNLG(answer, entities) {
222	// simple true-casing: uppercase all letters at the beginning of the sentence
223	// and after a period, question or exclamation mark
224	answer = answer.replace(/(^\| [.?!] )([a-z])/g, (_, prefix, letter) => prefix + letter.toUpperCase());	×
225
226	answer = answer.split(' ').map((token) => {	×
227	if (token in entities) {	×
228	if (token.startsWith('GENERIC_ENTITY_'))	×
229	return (entities[token].display \|\| entities[token].value);	×
230	return String(entities[token]);	×
231	}
232
233	// capitalize certain tokens that should be capitalized in English
234	if (MUST_CAPITALIZE_TOKEN.has(token))	×
235	return capitalize(token);	×
236	return token;	×
237	});
238	answer = detokenizeSentence(answer);	×
239
240	return answer;	×
241	}
242
243	// All the different forms in which MTurk workers write "no idea" for a sentence
244	// they don't understand
245	//
246	// This is usually empirically collected by looking at the results and finding
247	// sentences that don't validate or are too short
248	const NO_IDEA = [	52✔
249	'no idea', 'don\'t know', 'dont know', 'don\'t understand',
250	'dont understand', 'no clue',
251	'doesn\'t make sense', 'doesn\'t make any sense',
252	'doesnt make sense', 'doesnt make any sense'
253	];
254
255	// tb is terabyte in our dataset, tubercolosis in PPDB
256	// channel is TV/YouTube channel for us, river/waterway for PPDB
257	const PPDB_BLACKLIST = new Set(['tb', 'channel']);	52✔
258
259	// Check if a pair of word, paraphrase from PPDB should be considered a candidate
260	// for augmentation or not
261	function isValidParaphrasePair(word, paraphrase) {
262	if (PPDB_BLACKLIST.has(word))	474!
263	return false;	×
264	// ignore singular/plural relation and verb/gerund
265	if (paraphrase === word + 's' \|\| word === paraphrase + 's')	474!
266	return false;	×
267	if (paraphrase === word + 'ing' \|\| word === paraphrase + 'ing')	474!
268	return false;	×
269
270	// don't change the mode or tense of the verb
271	if (paraphrase.endsWith('ing') !== word.endsWith('ing'))	474✔
272	return false;	6✔
273	if (paraphrase.endsWith('ed') !== word.endsWith('ed'))	468✔
274	return false;	66✔
275	return true;	402✔
276	}
277
278	const CHANGE_SUBJECT_TEMPLATES = [	52✔
279	'ok , how about {}',
280	'how about {} instead',
281	'no {}',
282	'no i said {}',
283	'no , i said {}',
284	'i said {}',
285	'i want {} instead',
286	'no instead {}',
287	];
288
289	const SINGLE_DEVICE_TEMPLATES = [	52✔
290	['ask $device to $command', /^(?!(?:what\|when\|how\|who) )/],
291	['ask $device about $command', /^(?:what\|when\|how\|who) /],
292	['ask $device for $command', /^the /],
293	['ask $device $command', /^(?:what\|when\|how\|who) /],
294	['tell $device to $command', /^(?!(?:what\|when\|how\|who) )/],
295	['tell $device that $command', /^(?!(?:what\|when\|how\|who) )/],
296	['use $device to $command', /^(?!(?:what\|when\|how\|who) )/],
297	['use $device and $command', /^(?!(?:what\|when\|how\|who) )/],
298	['order $device to $command', /^(?!(?:what\|when\|how\|who) )/],
299	['$command from $device', null],
300	['$command using $device', null],
301	['$command in $device', null],
302	['$command by $device', null],
303	['$command with $device', null],
304	['talk to $device and $command', /^(?!(?:what\|when\|how\|who) )/],
305	['open $device and $command', /^(?!(?:what\|when\|how\|who) )/],
306	['launch $device and $command', /^(?!(?:what\|when\|how\|who) )/],
307	];
308
309	function pluralize(name) {
310	if (!name.includes(' ')) {	532✔
311	if (new Tag([name]).initial().tags[0] === 'NN')	98✔
312	return new Inflectors(name).toPlural();	96✔
313	return name;	2✔
314	} else {
315	const words = name.split(' ');	434✔
316	const tags = new Tag(words).initial().tags;	434✔
317	if (tags[tags.length - 1] !== 'NN')	434✔
318	return name;	276✔
319	else if (['VB', 'VBP', 'VBZ', 'VBD'].includes(tags[0]))	158✔
320	return name;	64✔
321	words[words.length - 1] = pluralize(words[words.length - 1]);	94✔
322	return words.join(' ');	94✔
323	}
324	}
325
326	function isGoodWord(word) {
327	// filter out words that cannot be in the dataset,
328	// because they would be either tokenized/preprocessed out or
329	// they are unlikely to be used with voice
330	return /^([a-zA-Z0-9-][a-zA-Z0-9.-]*\|'s\|,\|\?)$/.test(word);	3,862✔
331	}
332
333	function isGoodSentence(sentence) {
334	if (sentence.length < 3)	1,000✔
335	return false;	58✔
336	if (['.', ',', '?', '!', ' '].includes(sentence[0]))	942!
NEW 337	return false;	×
338	return !/^(the\|a\|has\|for\|title\|when\|me\|i\|so\|--\|to\|up\|and\|or\|of\|is\|are\|be\|being\|have\|an\|that\|this\|these\|those\|with\|as\|in\|on\|before\|after)$/.test(sentence);	942✔
339	}
340
341	function isGoodNumber(number) {
342	return /^([0-9]+)$/.test(number);	184✔
343	}
344
345
346	module.exports = {	52✔
347	postprocessSynthetic,
348	detokenize,
349	postprocessNLG,
350
351	pluralize,
352
353	ARGUMENT_NAME_OVERRIDES,
354
355	IGNORABLE_TOKENS,
356	ABBREVIATIONS: PROCESSED_ABBREVIATIONS,
357
358	NO_IDEA,
359
360	CHANGE_SUBJECT_TEMPLATES,
361	SINGLE_DEVICE_TEMPLATES,
362
363	isValidParaphrasePair,
364
365	isGoodWord,
366	isGoodSentence,
367	isGoodNumber
368	};

stanford-oval / genie-toolkit / 735

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous