1008

pending completion

Build # 1008

Build Type

Pull #207

travis-ci-com

Committed by

web-flow

Commit Message

Merge 0e0d197a5 into da825f499

Pull Request Pull Request #207: Wip/multilanguage 2

Run Details

5213 of 12183 branches covered (42.79%)

Branch coverage included in aggregate %.

36 of 43 new or added lines in 2 files covered. (83.72%)

2406 existing lines in 58 files now uncovered.

12858 of 23663 relevant lines covered (54.34%)

2272628.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.94

/lib/sentence-generator/runtime.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const assert = require('assert');

const List = require('./list');
const { uniform } = require('../utils/random');

const LogLevel = {
    NONE: 0,

    // log at the beginning and at the end of the generation for each depth, and notable events
    // such as particularly slow templates
    INFO: 1,

    // log each non-empty non terminal
    GENERATION: 2,

    // log all templates before generation
    DUMP_TEMPLATES: 3,

    // log information derived from the templates (such as the distance from the root)
    DUMP_DERIVED: 4,

    // log a lot of very redundant information during generation (can cause slowdowns)
    EVERYTHING: 5
};

// A numbered constant, eg. QUOTED_STRING_0 or NUMBER_1 or HASHTAG_3
// During generation, this constant is put in the program as a VarRef
// with an unique variable name.
class Constant {
    constructor(symbol, number, value) {
        this.symbol = symbol;
        this.number = number;
        this.value = value;
    }

    toString() {
        return `${this.symbol}_${this.number}`;
    }
}

class Placeholder {
    constructor(symbol, option) {
        this.symbol = symbol;
        this.option = option;
        assert(!option || option === 'const' || option === 'no-undefined');
    }

    toString() {
        return '${' + this.symbol + '}'; //'
    }
}

/**
 * A reference to a context.
 *
 * A context is an object that is passed as extra input to a semantic function
 * to affect its behavior. Grammar rules are only applied between identical (===) contexts.
 *
 * The "context" in this definition roughly corresponds to a dialogue context
 * (either a C: state, or a more general notion) but it need not be.
 *
 * "priv" is a value associated with the context that is only meaningful to the API caller
 * (DialogueGenerator). "info" is a value associated with the context that is meaningful
 * to the semantic function.
 * See {@link Grammar#_initializeContexts} for a longer explanation.
 */
class Context {
    constructor(priv, info) {
        this.priv = priv;
        this.info = info;
    }

    toString() {
        return `CTX[${this.info}]`;
    }

    static compatible(c1, c2) {
        return c1 === null || c2 === null || c1 === c2;
    }
    static meet(c1, c2) {
        if (c1 === null)
            return c2;
        else
            return c1;
    }
}

// A Derivation represents a sentence, possibly with placeholders,
// and a value, possibly with unspecified input parameters, that
// was computed at a certain point in the derivation tree
class Derivation {
    constructor(value, sentence, context = null, priority = 0) {
        this.value = value;
        if (value === undefined)
            throw new TypeError('Invalid value');
        this.context = context;
        assert(typeof this.context === 'object'); // incl. null
        this.sentence = sentence;
        this.priority = priority;
        assert(Number.isFinite(this.priority));
        assert(sentence instanceof List);

        this._flatSentence = null;
        this._hasPlaceholders = undefined;
    }

    hasPlaceholders() {
        if (this._hasPlaceholders !== undefined)
            return this._hasPlaceholders;

        let hasPlaceholders = false;
        this.sentence.traverse((child) => {
            hasPlaceholders = hasPlaceholders || child instanceof Placeholder;
        });
        return this._hasPlaceholders = hasPlaceholders;
    }

    hasPlaceholder(what) {
        let hasPlaceholder = false;
        this.sentence.traverse((child) => {
            hasPlaceholder = hasPlaceholder || (child instanceof Placeholder && child.symbol === what);
        });
        return hasPlaceholder;
    }

    get complete() {
        let v = !this.hasPlaceholders();
        return v;
    }

    toString() {
        if (this._flatSentence)
            return this._flatSentence;

        const flattened = [];
        this.sentence.traverse((el) => flattened.push(String(el)));
        return this._flatSentence = flattened.join(' ');
    }

    clone() {
        return new Derivation(this.value, this.sentence, this.context, this.priority);
    }

    replacePlaceholder(name, replacement, semanticAction, { isConstant, isUndefined = false, throwIfMissing = false }, rulePriority) {
        let newValue;
        let isDerivation;
        if (!(replacement instanceof Derivation)) {
            newValue = semanticAction(this.value);
            isDerivation = false;
        } else {
            assert(Context.compatible(this.context, replacement.context));
            newValue = semanticAction(this.value, replacement.value);
            isDerivation = true;
        }
        assert(newValue !== undefined);

        if (newValue === null) {
            /*if (!derivation.value.isVarRef || !derivation.value.name.startsWith('__const'))
                return null;*/
            /*if (throwIfMissing && this.hasPlaceholder(name)) {
                console.log('replace ' + name + ' in ' + this + ' with ' + derivation);
                console.log('values: ' + [this.value, derivation.value].join(' , '));
                throw new TypeError('???');
            }*/
            return null;
        }

        let newSentence = List.Nil;
        let newContext = this.context;
        let newPriority = this.priority + rulePriority;
        let found = false;
        let bad = false;
        this.sentence.traverse((child) => {
            if (child instanceof Placeholder && child.symbol === name) {
                if (child.option === 'no-undefined' && isUndefined)
                    bad = true;
                if (child.option === 'const' && !isConstant && !isUndefined)
                    bad = true;
                if (isDerivation) {
                    newSentence = List.concat(newSentence, replacement.sentence);
                    newContext = Context.meet(newContext, replacement.context);
                    newPriority += replacement.priority;
                } else {
                    newSentence = List.append(newSentence, replacement);
                }
                found = true;
            } else {
                newSentence = List.append(newSentence, child);
            }
        });

        if (!found || bad) {
            /*if (name === 'p_picture_url')
                console.log('no placeholder ' + name + ', have ' + String(this.sentence));
            if (throwIfMissing)
                throw new TypeError('???');
            */
            return null;
        }

        return new Derivation(newValue, newSentence, newContext, newPriority);
    }

    static combine(children, semanticAction, rulePriority) {
        if (children.length === 1) {
            assert(!(children[0] instanceof Context));
            if (children[0] instanceof Derivation) {
                let clone = children[0].clone();
                clone.value = semanticAction(children[0].value);
                if (clone.value === null)
                    return null;
                clone.priority += rulePriority;
                return clone;
            } else if (children[0] instanceof Placeholder) {
                let value = semanticAction();
                if (value === null)
                    return null;
                return new Derivation(value, List.singleton(children[0]), null, rulePriority);
            } else { // constant or terminal
                let value = semanticAction();
                if (value === null)
                    return null;
                return new Derivation(value, List.singleton(children[0]), null, rulePriority);
            }
        }

        let sentence = List.Nil;
        let values = [];
        let newContext = null;
        let newPriority = rulePriority;

        for (let child of children) {
            // does not go into the input sentence
            if (child instanceof Context) {
                newContext = child;
                values.push(child.info);
                continue;
            }

            if (typeof child === 'string' || child instanceof Constant || child instanceof Placeholder) { // terminal
                sentence = List.append(sentence, child);
            } else if (child instanceof Derivation) {
                assert(Context.compatible(newContext, child.context));
                newContext = Context.meet(newContext, child.context);
                newPriority += child.priority;
                values.push(child.value);
                sentence = List.concat(sentence, child.sentence);
            }
        }

        //console.log('combine: ' + children.join(' ++ '));
        //console.log('values: ' + values.join(' , '));

        let value = semanticAction(...values);
        assert(value !== undefined);
        if (value === null)
            return null;
        return new Derivation(value, sentence, newContext, newPriority);
    }
}


// Combination operators: use to create a semantic function that, given two child derivations,
// produces a new derivation

function simpleCombine(semanticAction, flag, topLevel = false) {
    return function(children, rulePriority) {

        const result = Derivation.combine(children, semanticAction, rulePriority);
        if (result === null)
            return null;
        if (flag) {
            if (flag.startsWith('!')) {
                if (result[flag.substring(1)])
                    return null;
            } else {
                if (!result[flag])
                    return null;
            }
        }
        if (!result.hasPlaceholders())
            return checkConstants(result, topLevel);
        else
            return result;
    };
}

function combineReplacePlaceholder(pname, semanticAction, options) {
    let f= function([c1, c2], rulePriority) {
        return c1.replacePlaceholder(pname, c2, semanticAction, options, rulePriority);
    };
    f.isReplacePlaceholder = true;
    return f;
}

function checkConstants(result, topLevel) {
    let constants = {};
    let bad = false;
    result.sentence.traverse((piece) => {
        if (!(piece instanceof Constant))
            return;
        if (piece.symbol in constants) {
            if (piece.number !== constants[piece.symbol] + 1) {
                bad = true;
                return;
            }
        } else {
            if (topLevel) {
                let min = 0;
                if (piece.number !== min) {
                    bad = true;
                    return;
                }
            }
        }
        constants[piece.symbol] = piece.number;
    });
    if (bad)
        return null;
    return result;
}

class NonTerminal {
    constructor(symbol) {
        this.symbol = symbol;
        this.index = -1;
    }

    toString() {
        return `NT[${this.symbol}]`;
    }
}

class Choice {
    constructor(choices) {
        this.choices = choices;
    }

    choose(rng) {
        return uniform(this.choices, rng);
    }

    toString() {
        return `C[${this.choices.join('|')}]`;
    }
}

//const everything = new Set;

module.exports = {
    LogLevel,

    Constant,
    Placeholder,
    Derivation,
    Context,
    NonTerminal,
    Choice,

    simpleCombine,
    combineReplacePlaceholder
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const assert = require('assert');	29✔
23
24	const List = require('./list');	29✔
25	const { uniform } = require('../utils/random');	29✔
26
27	const LogLevel = {	29✔
28	NONE: 0,
29
30	// log at the beginning and at the end of the generation for each depth, and notable events
31	// such as particularly slow templates
32	INFO: 1,
33
34	// log each non-empty non terminal
35	GENERATION: 2,
36
37	// log all templates before generation
38	DUMP_TEMPLATES: 3,
39
40	// log information derived from the templates (such as the distance from the root)
41	DUMP_DERIVED: 4,
42
43	// log a lot of very redundant information during generation (can cause slowdowns)
44	EVERYTHING: 5
45	};
46
47	// A numbered constant, eg. QUOTED_STRING_0 or NUMBER_1 or HASHTAG_3
48	// During generation, this constant is put in the program as a VarRef
49	// with an unique variable name.
50	class Constant {
51	constructor(symbol, number, value) {
52	this.symbol = symbol;	1,245✔
53	this.number = number;	1,245✔
54	this.value = value;	1,245✔
55	}
56
57	toString() {
58	return `${this.symbol}_${this.number}`;	1,522✔
59	}
60	}
61
62	class Placeholder {
63	constructor(symbol, option) {
64	this.symbol = symbol;	2,799✔
65	this.option = option;	2,799✔
66	assert(!option \|\| option === 'const' \|\| option === 'no-undefined');	2,799✔
67	}
68
69	toString() {
70	return '${' + this.symbol + '}'; //'	×
71	}
72	}
73
74	/**
75	* A reference to a context.
76	*
77	* A context is an object that is passed as extra input to a semantic function
78	* to affect its behavior. Grammar rules are only applied between identical (===) contexts.
79	*
80	* The "context" in this definition roughly corresponds to a dialogue context
81	* (either a C: state, or a more general notion) but it need not be.
82	*
83	* "priv" is a value associated with the context that is only meaningful to the API caller
84	* (DialogueGenerator). "info" is a value associated with the context that is meaningful
85	* to the semantic function.
86	* See {@link Grammar#_initializeContexts} for a longer explanation.
87	*/
88	class Context {
89	constructor(priv, info) {
90	this.priv = priv;	7,136✔
91	this.info = info;	7,136✔
92	}
93
94	toString() {
95	return `CTX[${this.info}]`;	×
96	}
97
98	static compatible(c1, c2) {
99	return c1 === null \|\| c2 === null \|\| c1 === c2;	2,016,612,419✔
100	}
101	static meet(c1, c2) {
102	if (c1 === null)	2,015,221,202✔
103	return c2;	939,021,788✔
104	else
105	return c1;	1,076,199,414✔
106	}
107	}
108
109	// A Derivation represents a sentence, possibly with placeholders,
110	// and a value, possibly with unspecified input parameters, that
111	// was computed at a certain point in the derivation tree
112	class Derivation {
113	constructor(value, sentence, context = null, priority = 0) {	3,352✔
114	this.value = value;	10,042,257✔
115	if (value === undefined)	10,042,257!
116	throw new TypeError('Invalid value');	×
117	this.context = context;	10,042,257✔
118	assert(typeof this.context === 'object'); // incl. null	10,042,257✔
119	this.sentence = sentence;	10,042,257✔
120	this.priority = priority;	10,042,257✔
121	assert(Number.isFinite(this.priority));	10,042,257✔
122	assert(sentence instanceof List);	10,042,257✔
123
124	this._flatSentence = null;	10,042,257✔
125	this._hasPlaceholders = undefined;	10,042,257✔
126	}
127
128	hasPlaceholders() {
129	if (this._hasPlaceholders !== undefined)	10,287,555✔
130	return this._hasPlaceholders;	288,792✔
131
132	let hasPlaceholders = false;	9,998,763✔
133	this.sentence.traverse((child) => {	9,998,763✔
134	hasPlaceholders = hasPlaceholders \|\| child instanceof Placeholder;	126,804,702✔
135	});
136	return this._hasPlaceholders = hasPlaceholders;	9,998,763✔
137	}
138
139	hasPlaceholder(what) {
140	let hasPlaceholder = false;	×
141	this.sentence.traverse((child) => {	×
142	hasPlaceholder = hasPlaceholder \|\| (child instanceof Placeholder && child.symbol === what);	×
143	});
144	return hasPlaceholder;	×
145	}
146
147	get complete() {
148	let v = !this.hasPlaceholders();	862,066✔
149	return v;	862,066✔
150	}
151
152	toString() {
153	if (this._flatSentence)	628,070✔
154	return this._flatSentence;	610,462✔
155
156	const flattened = [];	17,608✔
157	this.sentence.traverse((el) => flattened.push(String(el)));	194,743✔
158	return this._flatSentence = flattened.join(' ');	17,608✔
159	}
160
161	clone() {
162	return new Derivation(this.value, this.sentence, this.context, this.priority);	327,157✔
163	}
164
165	replacePlaceholder(name, replacement, semanticAction, { isConstant, isUndefined = false, throwIfMissing = false }, rulePriority) {	2,062,899✔
166	let newValue;
167	let isDerivation;
168	if (!(replacement instanceof Derivation)) {	1,032,486✔
169	newValue = semanticAction(this.value);	7,294✔
170	isDerivation = false;	7,294✔
171	} else {
172	assert(Context.compatible(this.context, replacement.context));	1,025,192✔
173	newValue = semanticAction(this.value, replacement.value);	1,025,192✔
174	isDerivation = true;	1,025,192✔
175	}
176	assert(newValue !== undefined);	1,032,486✔
177
178	if (newValue === null) {	1,032,486✔
179	/*if (!derivation.value.isVarRef \|\| !derivation.value.name.startsWith('__const'))
180	return null;*/
181	/*if (throwIfMissing && this.hasPlaceholder(name)) {
182	console.log('replace ' + name + ' in ' + this + ' with ' + derivation);
183	console.log('values: ' + [this.value, derivation.value].join(' , '));
184	throw new TypeError('???');
185	}*/
186	return null;	911,511✔
187	}
188
189	let newSentence = List.Nil;	120,975✔
190	let newContext = this.context;	120,975✔
191	let newPriority = this.priority + rulePriority;	120,975✔
192	let found = false;	120,975✔
193	let bad = false;	120,975✔
194	this.sentence.traverse((child) => {	120,975✔
195	if (child instanceof Placeholder && child.symbol === name) {	1,848,141✔
196	if (child.option === 'no-undefined' && isUndefined)	48,562!
UNCOV 197	bad = true;	×
198	if (child.option === 'const' && !isConstant && !isUndefined)	48,562!
199	bad = true;	×
200	if (isDerivation) {	48,562✔
201	newSentence = List.concat(newSentence, replacement.sentence);	47,384✔
202	newContext = Context.meet(newContext, replacement.context);	47,384✔
203	newPriority += replacement.priority;	47,384✔
204	} else {
205	newSentence = List.append(newSentence, replacement);	1,178✔
206	}
207	found = true;	48,562✔
208	} else {
209	newSentence = List.append(newSentence, child);	1,799,579✔
210	}
211	});
212
213	if (!found \|\| bad) {	120,975✔
214	/*if (name === 'p_picture_url')
215	console.log('no placeholder ' + name + ', have ' + String(this.sentence));
216	if (throwIfMissing)
217	throw new TypeError('???');
218	*/
219	return null;	72,413✔
220	}
221
222	return new Derivation(newValue, newSentence, newContext, newPriority);	48,562✔
223	}
224
225	static combine(children, semanticAction, rulePriority) {
226	if (children.length === 1) {	393,979,212✔
227	assert(!(children[0] instanceof Context));	336,361✔
228	if (children[0] instanceof Derivation) {	336,361✔
229	let clone = children[0].clone();	327,157✔
230	clone.value = semanticAction(children[0].value);	327,157✔
231	if (clone.value === null)	327,157✔
232	return null;	1,177✔
233	clone.priority += rulePriority;	325,980✔
234	return clone;	325,980✔
235	} else if (children[0] instanceof Placeholder) {	9,204!
UNCOV 236	let value = semanticAction();	×
UNCOV 237	if (value === null)	×
238	return null;	×
UNCOV 239	return new Derivation(value, List.singleton(children[0]), null, rulePriority);	×
240	} else { // constant or terminal
241	let value = semanticAction();	9,204✔
242	if (value === null)	9,204✔
243	return null;	46✔
244	return new Derivation(value, List.singleton(children[0]), null, rulePriority);	9,158✔
245	}
246	}
247
248	let sentence = List.Nil;	393,642,851✔
249	let values = [];	393,642,851✔
250	let newContext = null;	393,642,851✔
251	let newPriority = rulePriority;	393,642,851✔
252
253	for (let child of children) {	393,642,851✔
254	// does not go into the input sentence
255	if (child instanceof Context) {	1,347,469,622✔
256	newContext = child;	227,837,408✔
257	values.push(child.info);	227,837,408✔
258	continue;	227,837,408✔
259	}
260
261	if (typeof child === 'string' \|\| child instanceof Constant \|\| child instanceof Placeholder) { // terminal	1,119,632,214✔
262	sentence = List.append(sentence, child);	421,933,537✔
263	} else if (child instanceof Derivation) {	697,698,677✔
264	assert(Context.compatible(newContext, child.context));	697,698,668✔
265	newContext = Context.meet(newContext, child.context);	697,698,668✔
266	newPriority += child.priority;	697,698,668✔
267	values.push(child.value);	697,698,668✔
268	sentence = List.concat(sentence, child.sentence);	697,698,668✔
269	}
270	}
271
272	//console.log('combine: ' + children.join(' ++ '));
273	//console.log('values: ' + values.join(' , '));
274
275	let value = semanticAction(...values);	393,642,851✔
276	assert(value !== undefined);	393,642,851✔
277	if (value === null)	393,642,851✔
278	return null;	383,987,147✔
279	return new Derivation(value, sentence, newContext, newPriority);	9,655,704✔
280	}
281	}
282
283
284	// Combination operators: use to create a semantic function that, given two child derivations,
285	// produces a new derivation
286
287	function simpleCombine(semanticAction, flag, topLevel = false) {	35,728✔
288	return function(children, rulePriority) {	50,799✔
289
290	const result = Derivation.combine(children, semanticAction, rulePriority);	393,979,212✔
291	if (result === null)	393,979,212✔
292	return null;	383,988,370✔
293	if (flag) {	9,990,842✔
294	if (flag.startsWith('!')) {	862,066✔
295	if (result[flag.substring(1)])	770,905✔
296	return null;	731,366✔
297	} else {
298	if (!result[flag])	91,161✔
299	return null;	27,505✔
300	}
301	}
302	if (!result.hasPlaceholders())	9,231,971✔
303	return checkConstants(result, topLevel);	9,189,727✔
304	else
305	return result;	42,244✔
306	};
307	}
308
309	function combineReplacePlaceholder(pname, semanticAction, options) {
310	let f= function([c1, c2], rulePriority) {	7,224✔
311	return c1.replacePlaceholder(pname, c2, semanticAction, options, rulePriority);	1,032,486✔
312	};
313	f.isReplacePlaceholder = true;	7,224✔
314	return f;	7,224✔
315	}
316
317	function checkConstants(result, topLevel) {
318	let constants = {};	9,189,727✔
319	let bad = false;	9,189,727✔
320	result.sentence.traverse((piece) => {	9,189,727✔
321	if (!(piece instanceof Constant))	114,442,518✔
322	return;	109,998,553✔
323	if (piece.symbol in constants) {	4,443,965✔
324	if (piece.number !== constants[piece.symbol] + 1) {	442,882✔
325	bad = true;	72,534✔
326	return;	72,534✔
327	}
328	} else {
329	if (topLevel) {	4,001,083✔
330	let min = 0;	42,271✔
331	if (piece.number !== min) {	42,271✔
332	bad = true;	35,586✔
333	return;	35,586✔
334	}
335	}
336	}
337	constants[piece.symbol] = piece.number;	4,335,845✔
338	});
339	if (bad)	9,189,727✔
340	return null;	81,770✔
341	return result;	9,107,957✔
342	}
343
344	class NonTerminal {
345	constructor(symbol) {
346	this.symbol = symbol;	51,696✔
347	this.index = -1;	51,696✔
348	}
349
350	toString() {
351	return `NT[${this.symbol}]`;	75✔
352	}
353	}
354
355	class Choice {
356	constructor(choices) {
357	this.choices = choices;	8,720✔
358	}
359
360	choose(rng) {
361	return uniform(this.choices, rng);	148,732,838✔
362	}
363
364	toString() {
365	return `C[${this.choices.join('\|')}]`;	8✔
366	}
367	}
368
369	//const everything = new Set;
370
371	module.exports = {	29✔
372	LogLevel,
373
374	Constant,
375	Placeholder,
376	Derivation,
377	Context,
378	NonTerminal,
379	Choice,
380
381	simpleCombine,
382	combineReplacePlaceholder
383	};

stanford-oval / genie-toolkit / 1008

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous