1287

pending completion

Build # 1287

Build Type

Pull #322

travis-ci-com

Committed by

web-flow

Commit Message

Merge ada5e5e4a into 05146f7ad

Pull Request Pull Request #322: Another round of bug fixes

Run Details

6791 of 12876 branches covered (52.74%)

Branch coverage included in aggregate %.

59 of 90 new or added lines in 10 files covered. (65.56%)

24 existing lines in 8 files now uncovered.

15530 of 24740 relevant lines covered (62.77%)

1470933.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.67

/lib/i18n/default.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const BaseTokenizer = require('./tokenizer/base');

/**
 * Base class for all code that is specific to a certain natural language
 * in Genie.
 */
class DefaultLanguagePack {
    /**
     * Return an instance of the tokenizer used by this language.
     */
    getTokenizer() {
        if (this._tokenizer)
            return this._tokenizer;
        return this._tokenizer = new BaseTokenizer();
    }

    /**
     * Apply final touches to a newly generated synthetic sentence
     *
     * This function should correct coreferences, conjugations and other
     * grammar/readability issues that are too inconvenient to prevent
     * using the templates.
     */
    postprocessSynthetic(sentence, program, rng) {
        return sentence;
    }

    /**
     * Convert a tokenized sentence back into a correctly spaced, correctly
     * punctuated sentence.
     *
     * This is a low-level method called by {@link DefaultLanguagePack#detokenizeSentence}.
     * It can be used to detokenize one token at a time.
     */
    detokenize(sentence, prevtoken, token) {
        if (sentence && !this._NO_SPACE_TOKENS.has(token))
            sentence += ' ';
        sentence += token;
        return sentence;
    }

    /**
     * Convert a tokenized sentence back into a correctly spaced, correctly
     * punctuated sentence.
     *
     * This is used for sentences presented to an MTurk worker for paraphrasing,
     * and it is used for the agent replies before they are shown to the user.
     */
    detokenizeSentence(tokens) {
        let sentence = '';
        let prevToken = '';
        for (let token of tokens) {
            sentence = this.detokenize(sentence, prevToken, token);
            prevToken = token;
        }
        return sentence;
    }

    /**
     * Post-process a sentence generated by the neural NLG for display to
     * the user.
     *
     * This includes true-casing, detokenizing, and replacing entity tokens
     * with actual values.
     */
    postprocessNLG(answer, entities) {
        // simple true-casing: uppercase all letters at the beginning of the sentence
        // and after a period, question or exclamation mark
        answer = answer.replace(/(^| [.?!] )([a-z])/g, (_, prefix, letter) => prefix + letter.toUpperCase());

        answer = answer.split(' ').map((token) => {
            if (token in entities) {
                if (token.startsWith('GENERIC_ENTITY_'))
                    return (entities[token].display || entities[token].value);
                return String(entities[token]);
            }
            return token;
        });
        answer = this.detokenizeSentence(answer);

        return answer;
    }

    /**
     * Convert a word or phrase to plural form.
     *
     * This function should return `undefined` if there is no plural form
     * of the given phrase.
     */
    pluralize(phrase) {
        // no plural form
        return undefined;
    }

    /**
     * Convert a word or verb phrase to past tense.
     *
     * This function should return `undefined` if there is no past tense
     * of the given phrase.
     */
    toVerbPast(phrase) {
        // no past
        return undefined;
    }

    /**
     * Convert a phrase from the side of the user to the side of the agent.
     *
     * This function takes a phrase that talks about "my devices" (uttered by
     * the user) and converts to a phrase that talks about "your devices"
     * uttered by the agent.
     */
    toAgentSideUtterance(phrase) {
        // by default, no change
        return phrase;
    }

    /**
     * Filter out words that cannot be in the dataset, because they would be
     * either tokenized/preprocessed out or they are unlikely to be used with
     * voice.
     */
    isGoodWord(word) {
        // all words are good words
        return true;
    }

    /**
     * Filter out phrases that should not be used as a parameter on their own.
     *
     * This is mainly used to remove phrases that would be syntatically
     * ambiguous, and would not be immediately recognized as a parameter.
     * A good rule of thumb is to filter out all phrases that consist entirely
     * of stop words.
     */
    isGoodSentence(sentence) {
        // all sentences are good words
        return true;
    }

    /**
     * Check if a numeric phrase is valid for the given language.
     *
     * This covers ASCII digits as well as language-specific number systems,
     * like Arabic digits.
     */
    isGoodNumber(number) {
        return /^([0-9|\u0660-\u0669]+)$/.test(number);
    }

    /**
     * Check if a phrase looks like a person name.
     *
     * This is a coarse check that is used to override
     * {@link DefaultLanguagePack#isGoodWord} to account for foreign person
     * names and loan words.
     */
     isGoodPersonName(word) {
        return this.isGoodWord(word) || /^(\w+\s\w\s?\.)$/.test(word);
    }

    /**
     * Add a definite article ("the") to the given phrase.
     *
     * If the language has no concept of definite articles, this function
     * must return `undefined`.
     */
    addDefiniteArticle(phrase) {
        return undefined;
    }
}

/**
 * Override the canonical form of argument names for synthetic generation
 * (to generate filters and projections)
 *
 * More than one form can be provided for each argument name, in which case
 * all are used.
 */
DefaultLanguagePack.prototype.ARGUMENT_NAME_OVERRIDES = {
};

/**
 * Tokens that can be ignored in the names of entities, by entity type.
 *
 * This should cover abbreviations, prefixes and suffixes that are usually
 * omitted in colloquial speech.
 */
DefaultLanguagePack.prototype. IGNORABLE_TOKENS = {
    'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
    'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
    'org:freedesktop:app_id': ['gnome']
};

const ABBREVIATIONS = [];
const PROCESSED_ABBREVIATIONS = {};
for (let abbr of ABBREVIATIONS) {
    for (let variant of abbr)
        PROCESSED_ABBREVIATIONS[variant] = abbr;
}
/**
 * Interchangeable abbreviations for entity names
 *
 * Each entry in this array is a set (in array form) of abbreviations with the same
 * meaning; while expanding parameters, one of the possible forms is chosen at random
 *
 * Use this to fix tokenization inconsistencies in the entity database, to add
 * colloquial forms, and to add robustness to punctuation.
 */
DefaultLanguagePack.prototype.ABBREVIATIONS = PROCESSED_ABBREVIATIONS;

/**
 * Tokens that should not be preceded by a space.
 * This is used by the default {@link DefaultLanguagePack#detokenize}
 * implementation.
 */
DefaultLanguagePack.prototype._NO_SPACE_TOKENS = new Set(['.', ',', '?', '!', ':']);

/**
 * All the different forms in which MTurk workers write "no idea" for a sentence
 * they don't understand.
 *
 * This is usually empirically collected by looking at the results and finding
 * sentences that don't validate or are too short.
 */
DefaultLanguagePack.prototype.NO_IDEA = [];

DefaultLanguagePack.prototype.CHANGE_SUBJECT_TEMPLATES = [];

/**
 * Different ways to add an explicit reference to a skill name for a command.
 */
DefaultLanguagePack.prototype.SINGLE_DEVICE_TEMPLATES = [];

/**
 * A regular expression used to identify a definite article ("the") at the
 * beginning of a (tokenized) phrase.
 *
 * A language without definite articles should leave this to `undefined`.
 */
DefaultLanguagePack.prototype.DEFINITE_ARTICLE_REGEXP = undefined;

module.exports = DefaultLanguagePack;

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const BaseTokenizer = require('./tokenizer/base');	58✔
23
24	/**
25	* Base class for all code that is specific to a certain natural language
26	* in Genie.
27	*/
28	class DefaultLanguagePack {
29	/**
30	* Return an instance of the tokenizer used by this language.
31	*/
32	getTokenizer() {
33	if (this._tokenizer)	×
34	return this._tokenizer;	×
35	return this._tokenizer = new BaseTokenizer();	×
36	}
37
38	/**
39	* Apply final touches to a newly generated synthetic sentence
40	*
41	* This function should correct coreferences, conjugations and other
42	* grammar/readability issues that are too inconvenient to prevent
43	* using the templates.
44	*/
45	postprocessSynthetic(sentence, program, rng) {
46	return sentence;	×
47	}
48
49	/**
50	* Convert a tokenized sentence back into a correctly spaced, correctly
51	* punctuated sentence.
52	*
53	* This is a low-level method called by {@link DefaultLanguagePack#detokenizeSentence}.
54	* It can be used to detokenize one token at a time.
55	*/
56	detokenize(sentence, prevtoken, token) {
57	if (sentence && !this._NO_SPACE_TOKENS.has(token))	×
58	sentence += ' ';	×
59	sentence += token;	×
60	return sentence;	×
61	}
62
63	/**
64	* Convert a tokenized sentence back into a correctly spaced, correctly
65	* punctuated sentence.
66	*
67	* This is used for sentences presented to an MTurk worker for paraphrasing,
68	* and it is used for the agent replies before they are shown to the user.
69	*/
70	detokenizeSentence(tokens) {
71	let sentence = '';	56✔
72	let prevToken = '';	56✔
73	for (let token of tokens) {	56✔
74	sentence = this.detokenize(sentence, prevToken, token);	525✔
75	prevToken = token;	525✔
76	}
77	return sentence;	56✔
78	}
79
80	/**
81	* Post-process a sentence generated by the neural NLG for display to
82	* the user.
83	*
84	* This includes true-casing, detokenizing, and replacing entity tokens
85	* with actual values.
86	*/
87	postprocessNLG(answer, entities) {
88	// simple true-casing: uppercase all letters at the beginning of the sentence
89	// and after a period, question or exclamation mark
90	answer = answer.replace(/(^\| [.?!] )([a-z])/g, (_, prefix, letter) => prefix + letter.toUpperCase());	×
91
92	answer = answer.split(' ').map((token) => {	×
93	if (token in entities) {	×
94	if (token.startsWith('GENERIC_ENTITY_'))	×
95	return (entities[token].display \|\| entities[token].value);	×
96	return String(entities[token]);	×
97	}
98	return token;	×
99	});
100	answer = this.detokenizeSentence(answer);	×
101
102	return answer;	×
103	}
104
105	/**
106	* Convert a word or phrase to plural form.
107	*
108	* This function should return `undefined` if there is no plural form
109	* of the given phrase.
110	*/
111	pluralize(phrase) {
112	// no plural form
113	return undefined;	×
114	}
115
116	/**
117	* Convert a word or verb phrase to past tense.
118	*
119	* This function should return `undefined` if there is no past tense
120	* of the given phrase.
121	*/
122	toVerbPast(phrase) {
123	// no past
124	return undefined;	×
125	}
126
127	/**
128	* Convert a phrase from the side of the user to the side of the agent.
129	*
130	* This function takes a phrase that talks about "my devices" (uttered by
131	* the user) and converts to a phrase that talks about "your devices"
132	* uttered by the agent.
133	*/
134	toAgentSideUtterance(phrase) {
135	// by default, no change
NEW 136	return phrase;	×
137	}
138
139	/**
140	* Filter out words that cannot be in the dataset, because they would be
141	* either tokenized/preprocessed out or they are unlikely to be used with
142	* voice.
143	*/
144	isGoodWord(word) {
145	// all words are good words
146	return true;	×
147	}
148
149	/**
150	* Filter out phrases that should not be used as a parameter on their own.
151	*
152	* This is mainly used to remove phrases that would be syntatically
153	* ambiguous, and would not be immediately recognized as a parameter.
154	* A good rule of thumb is to filter out all phrases that consist entirely
155	* of stop words.
156	*/
157	isGoodSentence(sentence) {
158	// all sentences are good words
159	return true;	×
160	}
161
162	/**
163	* Check if a numeric phrase is valid for the given language.
164	*
165	* This covers ASCII digits as well as language-specific number systems,
166	* like Arabic digits.
167	*/
168	isGoodNumber(number) {
169	return /^([0-9\|\u0660-\u0669]+)$/.test(number);	×
170	}
171
172	/**
173	* Check if a phrase looks like a person name.
174	*
175	* This is a coarse check that is used to override
176	* {@link DefaultLanguagePack#isGoodWord} to account for foreign person
177	* names and loan words.
178	*/
179	isGoodPersonName(word) {
180	return this.isGoodWord(word) \|\| /^(\w+\s\w\s?\.)$/.test(word);	×
181	}
182
183	/**
184	* Add a definite article ("the") to the given phrase.
185	*
186	* If the language has no concept of definite articles, this function
187	* must return `undefined`.
188	*/
189	addDefiniteArticle(phrase) {
190	return undefined;	×
191	}
192	}
193
194	/**
195	* Override the canonical form of argument names for synthetic generation
196	* (to generate filters and projections)
197	*
198	* More than one form can be provided for each argument name, in which case
199	* all are used.
200	*/
201	DefaultLanguagePack.prototype.ARGUMENT_NAME_OVERRIDES = {	58✔
202	};
203
204	/**
205	* Tokens that can be ignored in the names of entities, by entity type.
206	*
207	* This should cover abbreviations, prefixes and suffixes that are usually
208	* omitted in colloquial speech.
209	*/
210	DefaultLanguagePack.prototype. IGNORABLE_TOKENS = {	58✔
211	'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
212	'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
213	'org:freedesktop:app_id': ['gnome']
214	};
215
216	const ABBREVIATIONS = [];	58✔
217	const PROCESSED_ABBREVIATIONS = {};	58✔
218	for (let abbr of ABBREVIATIONS) {	58✔
219	for (let variant of abbr)	×
220	PROCESSED_ABBREVIATIONS[variant] = abbr;	×
221	}
222	/**
223	* Interchangeable abbreviations for entity names
224	*
225	* Each entry in this array is a set (in array form) of abbreviations with the same
226	* meaning; while expanding parameters, one of the possible forms is chosen at random
227	*
228	* Use this to fix tokenization inconsistencies in the entity database, to add
229	* colloquial forms, and to add robustness to punctuation.
230	*/
231	DefaultLanguagePack.prototype.ABBREVIATIONS = PROCESSED_ABBREVIATIONS;	58✔
232
233	/**
234	* Tokens that should not be preceded by a space.
235	* This is used by the default {@link DefaultLanguagePack#detokenize}
236	* implementation.
237	*/
238	DefaultLanguagePack.prototype._NO_SPACE_TOKENS = new Set(['.', ',', '?', '!', ':']);	58✔
239
240	/**
241	* All the different forms in which MTurk workers write "no idea" for a sentence
242	* they don't understand.
243	*
244	* This is usually empirically collected by looking at the results and finding
245	* sentences that don't validate or are too short.
246	*/
247	DefaultLanguagePack.prototype.NO_IDEA = [];	58✔
248
249	DefaultLanguagePack.prototype.CHANGE_SUBJECT_TEMPLATES = [];	58✔
250
251	/**
252	* Different ways to add an explicit reference to a skill name for a command.
253	*/
254	DefaultLanguagePack.prototype.SINGLE_DEVICE_TEMPLATES = [];	58✔
255
256	/**
257	* A regular expression used to identify a definite article ("the") at the
258	* beginning of a (tokenized) phrase.
259	*
260	* A language without definite articles should leave this to `undefined`.
261	*/
262	DefaultLanguagePack.prototype.DEFINITE_ARTICLE_REGEXP = undefined;	58✔
263
264	module.exports = DefaultLanguagePack;	58✔

stanford-oval / genie-toolkit / 1287

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous