stanford-oval / genie-toolkit / 864

pending completion

Build # 864

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Merge pull request #175 from stanford-oval/wip/dialogue-new-annotations

Add new annotations

Run Details

1413 of 5434 branches covered (26.0%)

Branch coverage included in aggregate %.

76 of 170 new or added lines in 9 files covered. (44.71%)

2 existing lines in 2 files now uncovered.

4258 of 10544 relevant lines covered (40.38%)

133146.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.85

/lib/i18n/italian.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2018-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const assert = require('assert');

function replaceMeMy(sentence) {
    sentence = sentence.replace(/\b((?!(?:notifica|informa|invia a)\b)[a-zA-Z0-9]+) me\b/g, '$1 lui');

    return sentence.replace(/\b(io|me|mio|miei|mia|mie)\b/g, (what) => {
        switch(what) {
        case 'me':
        case 'io':
            return 'lui';
        case 'mio':
            return 'suo';
        case 'miei':
            return 'suoi';
        case 'mia':
            return 'sua';
        case 'mie':
            return 'sue';
        default:
            return what;
        }
    });
}

function postprocessSynthetic(sentence, program) {
    if (program.isProgram && program.principal !== null)
        sentence = replaceMeMy(sentence);

    return sentence.replace(/ (nuovi|nuove) (loro|suoi|miei|il|i|gli|le|un|una) /, (_new, what) => ` ${what} ${_new} `)
        //.replace(/ 's (my|their|his|her) /, ` 's `) //' // is there an equivalent in Italian? depends on the templates...
        .trim();
}

const ARGUMENT_NAME_OVERRIDES = {
    'picture_url': ['fotografia', 'foto', 'immagine'],

    'file_name': ['nome del file', 'nome'],
    'file_size': ['dimensione del file', 'dimensione'],
    'mime_type': ['tipo di file', 'tipo'],
};

const IGNORABLE_TOKENS = {
    'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
    'imgflip:meme_id': ['il'],
    'tt:currency_code': ['us'],
    'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
    'org:freedesktop:app_id': ['gnome']
};
const ABBREVIATIONS = [
    ['ltd', 'ltd.', 'limited'],
    ['corp', 'corp.', 'corporation'],
    ['l.l.c', 'llc'],
    ['inc.', 'inc', 'incorporated'],
    ['s.p.a', 'società per azioni'],
    ['s.r.l.', 'società a responsabilità limitata'],
    ['f.lli', 'fratelli'],
    ['&', 'e']
];
const PROCESSED_ABBREVIATIONS = {};
for (let abbr of ABBREVIATIONS) {
    for (let variant of abbr)
        PROCESSED_ABBREVIATIONS[variant] = abbr;
}

// tokens that are treated specially by the PTB tokenizer for English
// (and what they map to in properly punctuated English)
const SPECIAL_TOKENS = {
    '.': '.',
    ',': ',',
    '?': '?',

    // right/left round/curly/square bracket
    '-rrb-': ')',
    '-lrb-': ' (',
    '-rcb-': '}',
    '-lcb-': ' {',
    '-rsb-': ']',
    '-lsb-': ' [',
};

function detokenize(sentence, prevtoken, token) {
    if (token in SPECIAL_TOKENS) {
        sentence += SPECIAL_TOKENS[token];
    } else {
        if (sentence)
            sentence += ' ';
        sentence += token;
    }
    return sentence;
}

const NO_IDEA = [
    'no idea', 'non ho idea', 'non lo so', 'non so', 'non ho capito',
    'non capisco', 'boh', 'non ha senso', 'non si capisce'
];

const PPDB_BLACKLIST = new Set(['tb', 'canale']);
function isValidParaphrasePair(word, paraphrase) {
    if (PPDB_BLACKLIST.has(word))
        return false;

    // TODO: ignore singular/plural relation and verb/gerund
    // TODO: don't change the mode or tense of the verb
    // (conjugation rules are more complex in Italian than English
    // so this is not trivial...)
    return true;
}

const CHANGE_SUBJECT_TEMPLATES = [];
const SINGLE_DEVICE_TEMPLATES = [];

function pluralize(noun) {
    // TODO rules are complicated and full of exceptions...
    return undefined;
}

function toVerbPast(phrase) {
    // rules for past simple (passato remoto) are positively insane
    // no way in hell this is implementable
    return undefined;
}

function isGoodWord(word) {
    // filter out words that cannot be in the dataset,
    // because they would be either tokenized/preprocessed out or
    // they are unlikely to be used with voice
    return /^([\u00E0\u00C8\u00E8\u00E9\u00EC\u00F2\u00F9a-zA-Z0-9-][\u00E0\u00C8\u00E8\u00E9\u00EC\u00F2\u00F9a-zA-Z0-9.-]*|'s|,|\?)$/.test(word);
}

function isGoodSentence(sentence) {
    if (sentence.length < 3)
        return false;
    if (['.', ',', '?', '!', ' '].includes(sentence[0]))
        return false;
    return !/^(un|ha|per|titolo|quando|mi|i|modo|-|di|fino|sono|essere|avere|che|questa|questi|quelli|con|come|in|on|prima|dopo)$/.test(sentence);
}

function isGoodNumber(number) {
    return /^([0-9]+)$/.test(number);
}

function isGoodPersonName(word) {
    return isGoodWord(word) || /^(\w+\s\w\s?\.)$/.test(word);
}

const DEFINITE_ARTICLE_REGEXP = /(il|lo|la|i|gli|le) /;
function addDefiniteArticle(phrase) {
    const words = phrase.split(' ');
    assert(words.length > 0);

    const firstWord = words[0];

    // crude heuristic
    // to do better we would need a dictionary

    if (firstWord.endsWith('i')) {
        // masculine plural
        if (/^([haeiou]|s[^haeiou]|gn|ps|pn|z)/.test(firstWord))
            return 'gli ' + phrase;
        else
            return 'i ' + phrase;
    }
    if (firstWord.endsWith('a')) {
        // feminine singular
        if (/^haeiou/.test(firstWord))
            return 'l\' ' + phrase;
        else
            return 'la ' + phrase;
    }
    if (firstWord.endsWith('e')) {
        // feminine plural
        return 'le ' + phrase;
    }

    // all other cases, assume masculine singular
    if (/^haeiou/.test(firstWord))
        return 'l\' ' + phrase;
    else if (/^(s[^haeiou]|gn|ps|pn|z)/.test(firstWord))
        return 'lo ' + phrase;
    else
        return 'il ' + phrase;
}

module.exports = {
    postprocessSynthetic,
    detokenize,

    pluralize,
    toVerbPast,

    DEFINITE_ARTICLE_REGEXP,
    addDefiniteArticle,

    ARGUMENT_NAME_OVERRIDES,

    IGNORABLE_TOKENS,
    ABBREVIATIONS: PROCESSED_ABBREVIATIONS,

    NO_IDEA,
    CHANGE_SUBJECT_TEMPLATES,
    SINGLE_DEVICE_TEMPLATES,

    isValidParaphrasePair,

    isGoodWord,
    isGoodSentence,
    isGoodNumber,
    isGoodPersonName
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2018-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const assert = require('assert');	56✔
13
14	function replaceMeMy(sentence) {
15	sentence = sentence.replace(/\b((?!(?:notifica\|informa\|invia a)\b)[a-zA-Z0-9]+) me\b/g, '$1 lui');	×
16
17	return sentence.replace(/\b(io\|me\|mio\|miei\|mia\|mie)\b/g, (what) => {	×
18	switch(what) {	×
19	case 'me':
20	case 'io':
21	return 'lui';	×
22	case 'mio':
23	return 'suo';	×
24	case 'miei':
25	return 'suoi';	×
26	case 'mia':
27	return 'sua';	×
28	case 'mie':
29	return 'sue';	×
30	default:
31	return what;	×
32	}
33	});
34	}
35
36	function postprocessSynthetic(sentence, program) {
37	if (program.isProgram && program.principal !== null)	×
38	sentence = replaceMeMy(sentence);	×
39
40	return sentence.replace(/ (nuovi\|nuove) (loro\|suoi\|miei\|il\|i\|gli\|le\|un\|una) /, (_new, what) => ` ${what} ${_new} `)	×
41	//.replace(/ 's (my\|their\|his\|her) /, ` 's `) //' // is there an equivalent in Italian? depends on the templates...
42	.trim();
43	}
44
45	const ARGUMENT_NAME_OVERRIDES = {	56✔
46	'picture_url': ['fotografia', 'foto', 'immagine'],
47
48	'file_name': ['nome del file', 'nome'],
49	'file_size': ['dimensione del file', 'dimensione'],
50	'mime_type': ['tipo di file', 'tipo'],
51	};
52
53	const IGNORABLE_TOKENS = {	56✔
54	'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
55	'imgflip:meme_id': ['il'],
56	'tt:currency_code': ['us'],
57	'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
58	'org:freedesktop:app_id': ['gnome']
59	};
60	const ABBREVIATIONS = [	56✔
61	['ltd', 'ltd.', 'limited'],
62	['corp', 'corp.', 'corporation'],
63	['l.l.c', 'llc'],
64	['inc.', 'inc', 'incorporated'],
65	['s.p.a', 'società per azioni'],
66	['s.r.l.', 'società a responsabilità limitata'],
67	['f.lli', 'fratelli'],
68	['&', 'e']
69	];
70	const PROCESSED_ABBREVIATIONS = {};	56✔
71	for (let abbr of ABBREVIATIONS) {	56✔
72	for (let variant of abbr)	448✔
73	PROCESSED_ABBREVIATIONS[variant] = abbr;	1,064✔
74	}
75
76	// tokens that are treated specially by the PTB tokenizer for English
77	// (and what they map to in properly punctuated English)
78	const SPECIAL_TOKENS = {	56✔
79	'.': '.',
80	',': ',',
81	'?': '?',
82
83	// right/left round/curly/square bracket
84	'-rrb-': ')',
85	'-lrb-': ' (',
86	'-rcb-': '}',
87	'-lcb-': ' {',
88	'-rsb-': ']',
89	'-lsb-': ' [',
90	};
91
92	function detokenize(sentence, prevtoken, token) {
93	if (token in SPECIAL_TOKENS) {	×
94	sentence += SPECIAL_TOKENS[token];	×
95	} else {
96	if (sentence)	×
97	sentence += ' ';	×
98	sentence += token;	×
99	}
100	return sentence;	×
101	}
102
103	const NO_IDEA = [	56✔
104	'no idea', 'non ho idea', 'non lo so', 'non so', 'non ho capito',
105	'non capisco', 'boh', 'non ha senso', 'non si capisce'
106	];
107
108	const PPDB_BLACKLIST = new Set(['tb', 'canale']);	56✔
109	function isValidParaphrasePair(word, paraphrase) {
110	if (PPDB_BLACKLIST.has(word))	×
111	return false;	×
112
113	// TODO: ignore singular/plural relation and verb/gerund
114	// TODO: don't change the mode or tense of the verb
115	// (conjugation rules are more complex in Italian than English
116	// so this is not trivial...)
117	return true;	×
118	}
119
120	const CHANGE_SUBJECT_TEMPLATES = [];	56✔
121	const SINGLE_DEVICE_TEMPLATES = [];	56✔
122
123	function pluralize(noun) {
124	// TODO rules are complicated and full of exceptions...
125	return undefined;	×
126	}
127
128	function toVerbPast(phrase) {
129	// rules for past simple (passato remoto) are positively insane
130	// no way in hell this is implementable
131	return undefined;	×
132	}
133
134	function isGoodWord(word) {
135	// filter out words that cannot be in the dataset,
136	// because they would be either tokenized/preprocessed out or
137	// they are unlikely to be used with voice
138	return /^([\u00E0\u00C8\u00E8\u00E9\u00EC\u00F2\u00F9a-zA-Z0-9-][\u00E0\u00C8\u00E8\u00E9\u00EC\u00F2\u00F9a-zA-Z0-9.-]*\|'s\|,\|\?)$/.test(word);	×
139	}
140
141	function isGoodSentence(sentence) {
142	if (sentence.length < 3)	×
143	return false;	×
144	if (['.', ',', '?', '!', ' '].includes(sentence[0]))	×
145	return false;	×
146	return !/^(un\|ha\|per\|titolo\|quando\|mi\|i\|modo\|-\|di\|fino\|sono\|essere\|avere\|che\|questa\|questi\|quelli\|con\|come\|in\|on\|prima\|dopo)$/.test(sentence);	×
147	}
148
149	function isGoodNumber(number) {
150	return /^([0-9]+)$/.test(number);	×
151	}
152
153	function isGoodPersonName(word) {
154	return isGoodWord(word) \|\| /^(\w+\s\w\s?\.)$/.test(word);	×
155	}
156
157	const DEFINITE_ARTICLE_REGEXP = /(il\|lo\|la\|i\|gli\|le) /;	56✔
158	function addDefiniteArticle(phrase) {
NEW 159	const words = phrase.split(' ');	×
NEW 160	assert(words.length > 0);	×
161
NEW 162	const firstWord = words[0];	×
163
164	// crude heuristic
165	// to do better we would need a dictionary
166
NEW 167	if (firstWord.endsWith('i')) {	×
168	// masculine plural
NEW 169	if (/^([haeiou]\|s[^haeiou]\|gn\|ps\|pn\|z)/.test(firstWord))	×
NEW 170	return 'gli ' + phrase;	×
171	else
NEW 172	return 'i ' + phrase;	×
173	}
NEW 174	if (firstWord.endsWith('a')) {	×
175	// feminine singular
NEW 176	if (/^haeiou/.test(firstWord))	×
NEW 177	return 'l\' ' + phrase;	×
178	else
NEW 179	return 'la ' + phrase;	×
180	}
NEW 181	if (firstWord.endsWith('e')) {	×
182	// feminine plural
NEW 183	return 'le ' + phrase;	×
184	}
185
186	// all other cases, assume masculine singular
NEW 187	if (/^haeiou/.test(firstWord))	×
NEW 188	return 'l\' ' + phrase;	×
NEW 189	else if (/^(s[^haeiou]\|gn\|ps\|pn\|z)/.test(firstWord))	×
NEW 190	return 'lo ' + phrase;	×
191	else
NEW 192	return 'il ' + phrase;	×
193	}
194
195	module.exports = {	56✔
196	postprocessSynthetic,
197	detokenize,
198
199	pluralize,
200	toVerbPast,
201
202	DEFINITE_ARTICLE_REGEXP,
203	addDefiniteArticle,
204
205	ARGUMENT_NAME_OVERRIDES,
206
207	IGNORABLE_TOKENS,
208	ABBREVIATIONS: PROCESSED_ABBREVIATIONS,
209
210	NO_IDEA,
211	CHANGE_SUBJECT_TEMPLATES,
212	SINGLE_DEVICE_TEMPLATES,
213
214	isValidParaphrasePair,
215
216	isGoodWord,
217	isGoodSentence,
218	isGoodNumber,
219	isGoodPersonName
220	};

stanford-oval / genie-toolkit / 864

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous