2143

pending completion

Build # 2143

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Fix evaluation if the model predicts an empty string (#670)

Run Details

9948 of 20058 branches covered (49.6%)

Branch coverage included in aggregate %.

1 of 4 new or added lines in 1 file covered. (25.0%)

1 existing line in 1 file now uncovered.

23517 of 39544 relevant lines covered (59.47%)

500269.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.84

/lib/templates/dialogue_acts/search-questions.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>


import assert from 'assert';

import { Ast, } from 'thingtalk';

import * as C from '../ast_manip';

import {
    ContextInfo,
    makeAgentReply,
    makeSimpleState,
    addQuery,
    addQueryAndAction,
} from '../state_manip';
import {
    queryRefinement,
    refineFilterToAnswerQuestion
} from './refinement-helpers';
import {
    isValidSearchQuestion,
    isSimpleFilterExpression,
    addParametersFromContext
} from './common';

export type PreciseSearchQuestionAnswer = [Ast.Expression, Ast.Invocation|null, boolean];

export function preciseSearchQuestionAnswerKeyFn([table, action, hasYes] : PreciseSearchQuestionAnswer) {
    return {
        functionName: table.schema!.qualifiedName,
        actionFunctionName: action ? action.schema!.qualifiedName : null
    };
}

function isGoodSearchQuestion(ctx : ContextInfo, questions : C.ParamSlot[]) {
    const currentStmt = ctx.current!.stmt;
    if (!isValidSearchQuestion(currentStmt.lastQuery!, questions))
        return false;

    const ctxFilterTable = C.findFilterExpression(currentStmt.expression);
    if (!ctxFilterTable)
        return false;
    for (const q of questions) {
        if (C.filterUsesParam(ctxFilterTable.filter, q.name))
            return false;
    }
    return true;
}

function checkFilterPairForDisjunctiveQuestion(ctx : ContextInfo,
                                               f1 : C.FilterSlot,
                                               f2 : C.FilterSlot) : C.ParamSlot|null {
    if (!(f1.ast instanceof Ast.AtomBooleanExpression))
        return null;
    if (!(f2.ast instanceof Ast.AtomBooleanExpression))
        return null;
    if (!ctx.currentFunction!.is_list)
        return null;
    if (!C.isSameFunction(ctx.currentFunction!, f1.schema))
        return null;
    if (!C.isSameFunction(f1.schema, f2.schema))
        return null;
    if (f1.ast.name !== f2.ast.name)
        return null;
    if (!f1.ast.value.getType().equals(f2.ast.value.getType()))
        return null;
    if (f1.ast.value.equals(f2.ast.value))
        return null;
    const filterable = f1.schema.getArgument(f1.ast.name)!
        .getImplementationAnnotation<boolean>('filterable') ?? true;
    if (!filterable)
        return null;

    let good1 = false;
    let good2 = false;
    for (const result of ctx.results!) {
        const value = result.value[f1.ast.name];
        if (!value)
            return null;
        if (value.equals(f1.ast.value))
            good1 = true;
        if (value.equals(f2.ast.value))
            good2 = true;
        if (good1 && good2)
            break;
    }
    if (!good1 || !good2)
        return null;

    return { schema: f1.schema, type: f1.ptype, name: f1.ast.name,
        filterable, ast: new Ast.Value.VarRef(f1.ast.name) };
}

export function checkFilterPairForDisjunctiveQuestionWithConstant(ctx : ContextInfo,
                                                                  f1 : C.FilterSlot,
                                                                  c : Ast.Value) : C.ParamSlot|null {
    if (!(f1.ast instanceof Ast.AtomBooleanExpression))
        return null;
    const filterable = f1.schema.getArgument(f1.ast.name)!
        .getImplementationAnnotation<boolean>('filterable') ?? true;
    const pslot = { schema: f1.schema, name: f1.ast.name, type: f1.ptype, filterable,
        ast: new Ast.Value.VarRef(f1.ast.name) };
    const f2 = C.makeFilter(ctx.loader, pslot, f1.ast.operator, c);
    if (!f2)
        return null;
    return checkFilterPairForDisjunctiveQuestion(ctx, f1, f2);
}

function makeSearchQuestion(ctx : ContextInfo, questions : C.ParamSlot[]) {
    if (!isGoodSearchQuestion(ctx, questions))
        return null;

    if (questions.length === 0)
        return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_generic_search_question', null));

    if (questions.length === 1) {
        const currentStmt = ctx.current!.stmt;
        const type = currentStmt.lastQuery!.schema!.getArgument(questions[0].name)!.type;
        return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_search_question', questions.map((q) => q.name)), null, type);
    }

    return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_search_question', questions.map((q) => q.name)));
}

class AnswersQuestionVisitor extends Ast.NodeVisitor {
    answersQuestion = false;
    constructor(private questions : string[]) {
        super();
    }

    visitAtomBooleanExpression(atom : Ast.AtomBooleanExpression) {
        if (this.questions.some((q) => q === atom.name))
            this.answersQuestion = true;
        return true;
    }
    visitDontCareBooleanExpression(atom : Ast.DontCareBooleanExpression) {
        if (this.questions.some((q) => q === atom.name))
            this.answersQuestion = true;
        return true;
    }
}

/**
 * Check if the table filters on the parameters `questions` (effectively providing a constraint on question)
 */
function isQueryAnswerValidForQuestion(table : Ast.Expression, questions : string[]) {
    assert(Array.isArray(questions));
    const visitor = new AnswersQuestionVisitor(questions);
    table.visit(visitor);
    return visitor.answersQuestion;
}

function preciseSearchQuestionAnswer(ctx : ContextInfo, [answerTable, answerAction, _bool] : [Ast.Expression, Ast.Invocation|null, boolean]) {
    const questions = ctx.state.dialogueActParam as string[];
    assert(questions === null || (Array.isArray(questions) && questions.length > 0 && questions.every((q) => typeof q === 'string')));
    if (questions !== null && !isQueryAnswerValidForQuestion(answerTable, questions))
        return null;
    if (!(answerTable instanceof Ast.FilterExpression))
        return null;

    if (!C.isSameFunction(answerTable.schema!, ctx.currentFunction!))
        return null;
    const currentStmt = ctx.current!.stmt;
    const currentTable = currentStmt.expression;

    // TODO we need to push down the filter, if possible
    if (!isSimpleFilterExpression(answerTable))
        return null;

    if (answerAction !== null) {
        assert(answerAction instanceof Ast.Invocation);
        if (ctx.nextFunction !== null) {
            if (!C.isSameFunction(answerAction.schema!, ctx.nextFunction!))
                return null;

            // check that we don't fill the chain parameter through this path:
            // the chain parameter can only be filled if the agent shows the results
            for (const in_param of answerAction.in_params) {
                if (in_param.name === ctx.nextInfo!.chainParameter &&
                    !ctx.nextInfo!.chainParameterFilled)
                    return null;
            }

            answerAction = addParametersFromContext(answerAction, C.getInvocation(ctx.next!));
        }
    }

    const newTable = queryRefinement(currentTable, answerTable.filter, refineFilterToAnswerQuestion, null);
    if (newTable === null)
        return null;
    if (answerAction !== null)
        return addQueryAndAction(ctx, 'execute', newTable, answerAction, 'accepted');
    else
        return addQuery(ctx, 'execute', newTable, 'accepted');
}


function impreciseSearchQuestionAnswer(ctx : ContextInfo, answer : C.FilterSlot|Ast.Value|'dontcare') {
    const questions = ctx.state.dialogueActParam as string[];
    if (questions === null || questions.length !== 1)
        return null;
    assert(Array.isArray(questions) && questions.length > 0 && questions.every((q) => typeof q === 'string'));

    let answerFilter : C.FilterSlot;
    if (answer === 'dontcare') {
        answerFilter = {
            schema: ctx.currentFunction!,
            ptype: ctx.currentFunction!.getArgType(questions[0])!,
            ast: new Ast.BooleanExpression.DontCare(null, questions[0])
        };
    } else if (answer instanceof Ast.Value) {
        assert(questions.length === 1);
        assert(answer instanceof Ast.Value);

        const arg = ctx.currentFunction!.getArgument(questions[0])!;
        const pslot = { schema: ctx.currentFunction!,
            type: arg.type,
            filterable: arg.getImplementationAnnotation<boolean>('filterable') ?? true,
            name: questions[0],
            ast: new Ast.Value.VarRef(questions[0]) };
        const newFilter = C.makeFilter(ctx.loader, pslot, '==', answer);
        if (newFilter === null)
            return null;
        answerFilter = newFilter;
    } else {
        answerFilter = answer;
        let pname : string;
        if (answer.ast instanceof Ast.AndBooleanExpression) {
            assert(answer.ast.operands.length === 2);
            const [op1, op2] = answer.ast.operands;
            assert(op1 instanceof Ast.AtomBooleanExpression
                   && op2 instanceof Ast.AtomBooleanExpression);
            assert(op1.name === op2.name &&
                   op1.value.getType().equals(op2.value.getType()));
            pname = op1.name;
        } else if (answer.ast instanceof Ast.NotBooleanExpression) {
            const inner = answer.ast.expr;
            assert(inner instanceof Ast.AtomBooleanExpression ||
                   inner instanceof Ast.DontCareBooleanExpression);
            pname = inner.name;
        } else {
            assert(answer.ast instanceof Ast.AtomBooleanExpression ||
                   answer.ast instanceof Ast.DontCareBooleanExpression);
            pname = answer.ast.name;
        }
        if (!questions.some((q) => q === pname))
            return null;
    }

    const currentStmt = ctx.current!.stmt;
    const currentTable = currentStmt.expression;
    if (!C.checkFilter(currentTable, answerFilter))
        return null;

    const newTable = queryRefinement(currentTable, answerFilter.ast, refineFilterToAnswerQuestion, null);
    if (newTable === null)
        return null;
    return addQuery(ctx, 'execute', newTable, 'accepted');
}

export {
    checkFilterPairForDisjunctiveQuestion,
    makeSearchQuestion,
    preciseSearchQuestionAnswer,
    impreciseSearchQuestionAnswer
};

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21
22	import assert from 'assert';	7✔
23
24	import { Ast, } from 'thingtalk';	7✔
25
26	import * as C from '../ast_manip';	7✔
27
28	import {	7✔
29	ContextInfo,
30	makeAgentReply,
31	makeSimpleState,
32	addQuery,
33	addQueryAndAction,
34	} from '../state_manip';
35	import {	7✔
36	queryRefinement,
37	refineFilterToAnswerQuestion
38	} from './refinement-helpers';
39	import {	7✔
40	isValidSearchQuestion,
41	isSimpleFilterExpression,
42	addParametersFromContext
43	} from './common';
44
45	export type PreciseSearchQuestionAnswer = [Ast.Expression, Ast.Invocation\|null, boolean];
46
47	export function preciseSearchQuestionAnswerKeyFn([table, action, hasYes] : PreciseSearchQuestionAnswer) {	7✔
48	return {	86,150✔
49	functionName: table.schema!.qualifiedName,
50	actionFunctionName: action ? action.schema!.qualifiedName : null	86,150✔
51	};
52	}
53
54	function isGoodSearchQuestion(ctx : ContextInfo, questions : C.ParamSlot[]) {
55	const currentStmt = ctx.current!.stmt;	48,070✔
56	if (!isValidSearchQuestion(currentStmt.lastQuery!, questions))	48,070✔
57	return false;	24,812✔
58
59	const ctxFilterTable = C.findFilterExpression(currentStmt.expression);	23,258✔
60	if (!ctxFilterTable)	23,258✔
61	return false;	14,908✔
62	for (const q of questions) {	8,350✔
63	if (C.filterUsesParam(ctxFilterTable.filter, q.name))	8,798✔
64	return false;	7,614✔
65	}
66	return true;	736✔
67	}
68
69	function checkFilterPairForDisjunctiveQuestion(ctx : ContextInfo,
70	f1 : C.FilterSlot,
71	f2 : C.FilterSlot) : C.ParamSlot\|null {
72	if (!(f1.ast instanceof Ast.AtomBooleanExpression))	×
73	return null;	×
74	if (!(f2.ast instanceof Ast.AtomBooleanExpression))	×
75	return null;	×
76	if (!ctx.currentFunction!.is_list)	×
77	return null;	×
78	if (!C.isSameFunction(ctx.currentFunction!, f1.schema))	×
79	return null;	×
80	if (!C.isSameFunction(f1.schema, f2.schema))	×
81	return null;	×
82	if (f1.ast.name !== f2.ast.name)	×
83	return null;	×
84	if (!f1.ast.value.getType().equals(f2.ast.value.getType()))	×
85	return null;	×
86	if (f1.ast.value.equals(f2.ast.value))	×
87	return null;	×
88	const filterable = f1.schema.getArgument(f1.ast.name)!	×
89	.getImplementationAnnotation<boolean>('filterable') ?? true;	×
90	if (!filterable)	×
91	return null;	×
92
93	let good1 = false;	×
94	let good2 = false;	×
95	for (const result of ctx.results!) {	×
96	const value = result.value[f1.ast.name];	×
97	if (!value)	×
98	return null;	×
99	if (value.equals(f1.ast.value))	×
100	good1 = true;	×
101	if (value.equals(f2.ast.value))	×
102	good2 = true;	×
103	if (good1 && good2)	×
104	break;	×
105	}
106	if (!good1 \|\| !good2)	×
107	return null;	×
108
109	return { schema: f1.schema, type: f1.ptype, name: f1.ast.name,	×
110	filterable, ast: new Ast.Value.VarRef(f1.ast.name) };
111	}
112
113	export function checkFilterPairForDisjunctiveQuestionWithConstant(ctx : ContextInfo,	7✔
114	f1 : C.FilterSlot,
115	c : Ast.Value) : C.ParamSlot\|null {
116	if (!(f1.ast instanceof Ast.AtomBooleanExpression))	×
117	return null;	×
118	const filterable = f1.schema.getArgument(f1.ast.name)!	×
119	.getImplementationAnnotation<boolean>('filterable') ?? true;	×
120	const pslot = { schema: f1.schema, name: f1.ast.name, type: f1.ptype, filterable,	×
121	ast: new Ast.Value.VarRef(f1.ast.name) };
122	const f2 = C.makeFilter(ctx.loader, pslot, f1.ast.operator, c);	×
123	if (!f2)	×
124	return null;	×
125	return checkFilterPairForDisjunctiveQuestion(ctx, f1, f2);	×
126	}
127
128	function makeSearchQuestion(ctx : ContextInfo, questions : C.ParamSlot[]) {
129	if (!isGoodSearchQuestion(ctx, questions))	48,070✔
130	return null;	47,334✔
131
132	if (questions.length === 0)	736!
133	return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_generic_search_question', null));	×
134
135	if (questions.length === 1) {	736✔
136	const currentStmt = ctx.current!.stmt;	344✔
137	const type = currentStmt.lastQuery!.schema!.getArgument(questions[0].name)!.type;	344✔
138	return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_search_question', questions.map((q) => q.name)), null, type);	344✔
139	}
140
141	return makeAgentReply(ctx, makeSimpleState(ctx, 'sys_search_question', questions.map((q) => q.name)));	784✔
142	}
143
144	class AnswersQuestionVisitor extends Ast.NodeVisitor {
145	answersQuestion = false;	96,798✔
146	constructor(private questions : string[]) {	96,798✔
147	super();	96,798✔
148	}
149
150	visitAtomBooleanExpression(atom : Ast.AtomBooleanExpression) {
151	if (this.questions.some((q) => q === atom.name))	256,464✔
152	this.answersQuestion = true;	13,382✔
153	return true;	165,893✔
154	}
155	visitDontCareBooleanExpression(atom : Ast.DontCareBooleanExpression) {
156	if (this.questions.some((q) => q === atom.name))	×
157	this.answersQuestion = true;	×
158	return true;	×
159	}
160	}
161
162	/**
163	* Check if the table filters on the parameters `questions` (effectively providing a constraint on question)
164	*/
165	function isQueryAnswerValidForQuestion(table : Ast.Expression, questions : string[]) {
166	assert(Array.isArray(questions));	96,798✔
167	const visitor = new AnswersQuestionVisitor(questions);	96,798✔
168	table.visit(visitor);	96,798✔
169	return visitor.answersQuestion;	96,798✔
170	}
171
172	function preciseSearchQuestionAnswer(ctx : ContextInfo, [answerTable, answerAction, _bool] : [Ast.Expression, Ast.Invocation\|null, boolean]) {
173	const questions = ctx.state.dialogueActParam as string[];	96,798✔
174	assert(questions === null \|\| (Array.isArray(questions) && questions.length > 0 && questions.every((q) => typeof q === 'string')));	149,898✔
175	if (questions !== null && !isQueryAnswerValidForQuestion(answerTable, questions))	96,798✔
176	return null;	86,523✔
177	if (!(answerTable instanceof Ast.FilterExpression))	10,275!
UNCOV 178	return null;	×
179
180	if (!C.isSameFunction(answerTable.schema!, ctx.currentFunction!))	10,275!
181	return null;	×
182	const currentStmt = ctx.current!.stmt;	10,275✔
183	const currentTable = currentStmt.expression;	10,275✔
184
185	// TODO we need to push down the filter, if possible
186	if (!isSimpleFilterExpression(answerTable))	10,275!
187	return null;	×
188
189	if (answerAction !== null) {	10,275✔
190	assert(answerAction instanceof Ast.Invocation);	3✔
191	if (ctx.nextFunction !== null) {	3✔
192	if (!C.isSameFunction(answerAction.schema!, ctx.nextFunction!))	2!
193	return null;	×
194
195	// check that we don't fill the chain parameter through this path:
196	// the chain parameter can only be filled if the agent shows the results
197	for (const in_param of answerAction.in_params) {	2✔
198	if (in_param.name === ctx.nextInfo!.chainParameter &&	8✔
199	!ctx.nextInfo!.chainParameterFilled)
200	return null;	2✔
201	}
202
203	answerAction = addParametersFromContext(answerAction, C.getInvocation(ctx.next!));	×
204	}
205	}
206
207	const newTable = queryRefinement(currentTable, answerTable.filter, refineFilterToAnswerQuestion, null);	10,273✔
208	if (newTable === null)	10,273✔
209	return null;	738✔
210	if (answerAction !== null)	9,535✔
211	return addQueryAndAction(ctx, 'execute', newTable, answerAction, 'accepted');	1✔
212	else
213	return addQuery(ctx, 'execute', newTable, 'accepted');	9,534✔
214	}
215
216
217	function impreciseSearchQuestionAnswer(ctx : ContextInfo, answer : C.FilterSlot\|Ast.Value\|'dontcare') {
218	const questions = ctx.state.dialogueActParam as string[];	662,963✔
219	if (questions === null \|\| questions.length !== 1)	662,963✔
220	return null;	309,103✔
221	assert(Array.isArray(questions) && questions.length > 0 && questions.every((q) => typeof q === 'string'));	353,860✔
222
223	let answerFilter : C.FilterSlot;
224	if (answer === 'dontcare') {	353,860!
225	answerFilter = {	×
226	schema: ctx.currentFunction!,
227	ptype: ctx.currentFunction!.getArgType(questions[0])!,
228	ast: new Ast.BooleanExpression.DontCare(null, questions[0])
229	};
230	} else if (answer instanceof Ast.Value) {	353,860✔
231	assert(questions.length === 1);	87,287✔
232	assert(answer instanceof Ast.Value);	87,287✔
233
234	const arg = ctx.currentFunction!.getArgument(questions[0])!;	87,287✔
235	const pslot = { schema: ctx.currentFunction!,	87,287✔
236	type: arg.type,
237	filterable: arg.getImplementationAnnotation<boolean>('filterable') ?? true,	261,861!
238	name: questions[0],
239	ast: new Ast.Value.VarRef(questions[0]) };
240	const newFilter = C.makeFilter(ctx.loader, pslot, '==', answer);	87,287✔
241	if (newFilter === null)	87,287✔
242	return null;	85,837✔
243	answerFilter = newFilter;	1,450✔
244	} else {
245	answerFilter = answer;	266,573✔
246	let pname : string;
247	if (answer.ast instanceof Ast.AndBooleanExpression) {	266,573✔
248	assert(answer.ast.operands.length === 2);	97,410✔
249	const [op1, op2] = answer.ast.operands;	97,410✔
250	assert(op1 instanceof Ast.AtomBooleanExpression	97,410✔
251	&& op2 instanceof Ast.AtomBooleanExpression);
252	assert(op1.name === op2.name &&	97,410✔
253	op1.value.getType().equals(op2.value.getType()));
254	pname = op1.name;	97,410✔
255	} else if (answer.ast instanceof Ast.NotBooleanExpression) {	169,163!
256	const inner = answer.ast.expr;	×
257	assert(inner instanceof Ast.AtomBooleanExpression \|\|	×
258	inner instanceof Ast.DontCareBooleanExpression);
259	pname = inner.name;	×
260	} else {
261	assert(answer.ast instanceof Ast.AtomBooleanExpression \|\|	169,163!
262	answer.ast instanceof Ast.DontCareBooleanExpression);
263	pname = answer.ast.name;	169,163✔
264	}
265	if (!questions.some((q) => q === pname))	266,573✔
266	return null;	254,967✔
267	}
268
269	const currentStmt = ctx.current!.stmt;	13,056✔
270	const currentTable = currentStmt.expression;	13,056✔
271	if (!C.checkFilter(currentTable, answerFilter))	13,056✔
272	return null;	5,814✔
273
274	const newTable = queryRefinement(currentTable, answerFilter.ast, refineFilterToAnswerQuestion, null);	7,242✔
275	if (newTable === null)	7,242!
276	return null;	×
277	return addQuery(ctx, 'execute', newTable, 'accepted');	7,242✔
278	}
279
280	export {
281	checkFilterPairForDisjunctiveQuestion,	7✔
282	makeSearchQuestion,	7✔
283	preciseSearchQuestionAnswer,	7✔
284	impreciseSearchQuestionAnswer	7✔
285	};

stanford-oval / genie-toolkit / 2143

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous