1660

pending completion

Build # 1660

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Merge pull request #421 from stanford-oval/wip/confidence

Confidence-based dispatch

Run Details

10837 of 23282 branches covered (46.55%)

Branch coverage included in aggregate %.

191 of 312 new or added lines in 8 files covered. (61.22%)

21 existing lines in 12 files now uncovered.

25387 of 43912 relevant lines covered (57.81%)

522452.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.82

/lib/utils/thingtalk/index.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>


import assert from 'assert';
import yaml from 'js-yaml';
import util from 'util';
import * as fs from 'fs';

import { Ast, SchemaRetriever } from 'thingtalk';

import SimulationDialogueAgent, { SimulationDialogueAgentOptions } from '../../dialogue-agent/simulator/simulation_dialogue_agent';
import { extractConstants, createConstants } from './constants';
export * from './describe';
export * from './syntax';
export * from './dialogue_state_utils';
import { computePrediction } from './dialogue_state_utils';
export * from './example-utils';
// reexport clean from misc-utils
import { clean } from '../misc-utils';
export { clean };

export type Input = Ast.Input;
export type DialogueState = Ast.DialogueState;
export type Simulator = SimulationDialogueAgent;
export type SimulatorOptions = SimulationDialogueAgentOptions;

export {
    extractConstants,
    createConstants,
};

function validateState(state : Ast.DialogueState, forTarget : 'user'|'agent') : void {
    if (forTarget === 'user') {
        // check that there are no 'proposed' items
        // (should be executed, 'accepted' or 'confirmed')
        for (const item of state.history)
            assert(item.confirm !== 'proposed');
    } else {
        // check that there are no 'confirmed' items that were not executed
        // TODO: if we add "intermediate_context" capabilities to the state machine
        // we can relax this restriction
        for (const item of state.history)
            assert(item.confirm !== 'confirmed' || item.results !== null);
    }
}

export function createSimulator(options : SimulationDialogueAgentOptions) : SimulationDialogueAgent {
    const tpClient = options.thingpediaClient;
    if (!options.schemaRetriever)
        options.schemaRetriever = new SchemaRetriever(tpClient!, null, true);
    return new SimulationDialogueAgent(options);
}

interface PolicyManifest {
    name : string;
    terminalAct : string;
    dialogueActs : {
        user : Set<string>;
        agent : Set<string>;
        withParam : Set<string>;
    };
}

class StateValidator {
    private _policyManifest : string|undefined;
    private _policy : PolicyManifest|null;

    constructor(policyManifest : string|undefined) {
        this._policyManifest = policyManifest;
        this._policy = null;
    }

    async load() : Promise<void> {
        if (!this._policyManifest)
            return;
        const buffer = await util.promisify(fs.readFile)(this._policyManifest, { encoding: 'utf8' });
        const policy = yaml.safeLoad(buffer) as any;

        this._policy = {
            name: policy.name,
            terminalAct: policy.terminalAct,
            dialogueActs: {
                user: new Set(policy.dialogueActs.user),
                agent: new Set(policy.dialogueActs.agent),
                withParam: new Set(policy.dialogueActs.withParam)
            }
        };
    }

    validateUser(state : Ast.DialogueState) : void {
        validateState(state, 'user');

        if (!this._policy)
            return;
        assert.strictEqual(state.policy, this._policy.name);
        assert(this._policy.dialogueActs.user.has(state.dialogueAct), `Invalid user dialogue act ${state.dialogueAct}`);
        // if and only if
        assert((state.dialogueActParam !== null) === (this._policy.dialogueActs.withParam.has(state.dialogueAct)));
    }

    validateAgent(state : Ast.DialogueState) : void {
        validateState(state, 'agent');

        if (!this._policy)
            return;
        assert.strictEqual(state.policy, this._policy.name);
        assert(this._policy.dialogueActs.agent.has(state.dialogueAct), `Invalid agent dialogue act ${state.dialogueAct}`);
        // if and only if
        assert((state.dialogueActParam !== null) === (this._policy.dialogueActs.withParam.has(state.dialogueAct)));
    }
}

export function createStateValidator(policyManifest ?: string) : StateValidator {
    return new StateValidator(policyManifest);
}

interface DialoguePolicy {
    handleAnswer(state : Ast.DialogueState, value : Ast.Value) : Promise<Ast.DialogueState|null>;
}

export async function inputToDialogueState(policy : DialoguePolicy,
                                           context : Ast.DialogueState|null,
                                           input : Ast.Input) : Promise<Ast.DialogueState|null> {
    if (input instanceof Ast.ControlCommand) {
        if (context === null)
            return null;

        if (input.intent instanceof Ast.SpecialControlIntent) {
            switch (input.intent.type) {
            case 'yes':
            case 'no': {
                const value = new Ast.BooleanValue(input.intent.type === 'yes');
                const handled = await policy.handleAnswer(context, value);
                if (!handled)
                    return null;
                return computePrediction(context, handled, 'user');
            }
            default:
                return null;
            }
        }
        if (input.intent instanceof Ast.ChoiceControlIntent)
            return null;

        if (input.intent instanceof Ast.AnswerControlIntent) {
            const handled = await policy.handleAnswer(context, input.intent.value);
            if (!handled)
                return null;
            return computePrediction(context, handled, 'user');
        }

        throw new TypeError(`Unrecognized bookkeeping intent`);
    } else if (input instanceof Ast.Program) {
        // convert thingtalk programs to dialogue states so we can use "\t" without too much typing
        const prediction = new Ast.DialogueState(null, 'org.thingpedia.dialogue.transaction', 'execute', null, []);
        for (const stmt of input.statements) {
            if (stmt instanceof Ast.Assignment)
                throw new Error(`Unsupported: assignment statement`);
            prediction.history.push(new Ast.DialogueHistoryItem(null, stmt, null, 'accepted'));
        }
        return prediction;
    }

    assert(input instanceof Ast.DialogueState);
    return input;
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21
22	import assert from 'assert';	61✔
23	import yaml from 'js-yaml';	61✔
24	import util from 'util';	61✔
25	import * as fs from 'fs';	61✔
26
27	import { Ast, SchemaRetriever } from 'thingtalk';	61✔
28
29	import SimulationDialogueAgent, { SimulationDialogueAgentOptions } from '../../dialogue-agent/simulator/simulation_dialogue_agent';	61✔
30	import { extractConstants, createConstants } from './constants';	61✔
31	export * from './describe';	61✔
32	export * from './syntax';	61✔
33	export * from './dialogue_state_utils';	61✔
34	import { computePrediction } from './dialogue_state_utils';	61✔
35	export * from './example-utils';	61✔
36	// reexport clean from misc-utils
37	import { clean } from '../misc-utils';	61✔
38	export { clean };	1,698✔
39
40	export type Input = Ast.Input;
41	export type DialogueState = Ast.DialogueState;
42	export type Simulator = SimulationDialogueAgent;
43	export type SimulatorOptions = SimulationDialogueAgentOptions;
44
45	export {
46	extractConstants,	130✔
47	createConstants,	1,263✔
48	};
49
50	function validateState(state : Ast.DialogueState, forTarget : 'user'\|'agent') : void {
51	if (forTarget === 'user') {	39,721✔
52	// check that there are no 'proposed' items
53	// (should be executed, 'accepted' or 'confirmed')
54	for (const item of state.history)	24,223✔
55	assert(item.confirm !== 'proposed');	46,989✔
56	} else {
57	// check that there are no 'confirmed' items that were not executed
58	// TODO: if we add "intermediate_context" capabilities to the state machine
59	// we can relax this restriction
60	for (const item of state.history)	15,498✔
61	assert(item.confirm !== 'confirmed' \|\| item.results !== null);	28,975✔
62	}
63	}
64
65	export function createSimulator(options : SimulationDialogueAgentOptions) : SimulationDialogueAgent {	61✔
66	const tpClient = options.thingpediaClient;	3✔
67	if (!options.schemaRetriever)	3!
68	options.schemaRetriever = new SchemaRetriever(tpClient!, null, true);	3✔
69	return new SimulationDialogueAgent(options);	3✔
70	}
71
72	interface PolicyManifest {
73	name : string;
74	terminalAct : string;
75	dialogueActs : {
76	user : Set<string>;
77	agent : Set<string>;
78	withParam : Set<string>;
79	};
80	}
81
82	class StateValidator {
83	private _policyManifest : string\|undefined;
84	private _policy : PolicyManifest\|null;
85
86	constructor(policyManifest : string\|undefined) {
87	this._policyManifest = policyManifest;	3✔
88	this._policy = null;	3✔
89	}
90
91	async load() : Promise<void> {
92	if (!this._policyManifest)	3✔
93	return;	1✔
94	const buffer = await util.promisify(fs.readFile)(this._policyManifest, { encoding: 'utf8' });	2✔
95	const policy = yaml.safeLoad(buffer) as any;	2✔
96
97	this._policy = {	2✔
98	name: policy.name,
99	terminalAct: policy.terminalAct,
100	dialogueActs: {
101	user: new Set(policy.dialogueActs.user),
102	agent: new Set(policy.dialogueActs.agent),
103	withParam: new Set(policy.dialogueActs.withParam)
104	}
105	};
106	}
107
108	validateUser(state : Ast.DialogueState) : void {
109	validateState(state, 'user');	24,223✔
110
111	if (!this._policy)	24,223✔
112	return;	4,073✔
113	assert.strictEqual(state.policy, this._policy.name);	20,150✔
114	assert(this._policy.dialogueActs.user.has(state.dialogueAct), `Invalid user dialogue act ${state.dialogueAct}`);	20,150✔
115	// if and only if
116	assert((state.dialogueActParam !== null) === (this._policy.dialogueActs.withParam.has(state.dialogueAct)));	20,150✔
117	}
118
119	validateAgent(state : Ast.DialogueState) : void {
120	validateState(state, 'agent');	15,498✔
121
122	if (!this._policy)	15,498✔
123	return;	2,162✔
124	assert.strictEqual(state.policy, this._policy.name);	13,336✔
125	assert(this._policy.dialogueActs.agent.has(state.dialogueAct), `Invalid agent dialogue act ${state.dialogueAct}`);	13,336✔
126	// if and only if
127	assert((state.dialogueActParam !== null) === (this._policy.dialogueActs.withParam.has(state.dialogueAct)));	13,336✔
128	}
129	}
130
131	export function createStateValidator(policyManifest ?: string) : StateValidator {	61✔
132	return new StateValidator(policyManifest);	3✔
133	}
134
135	interface DialoguePolicy {
136	handleAnswer(state : Ast.DialogueState, value : Ast.Value) : Promise<Ast.DialogueState\|null>;
137	}
138
139	export async function inputToDialogueState(policy : DialoguePolicy,	61✔
140	context : Ast.DialogueState\|null,
141	input : Ast.Input) : Promise<Ast.DialogueState\|null> {
142	if (input instanceof Ast.ControlCommand) {	72✔
143	if (context === null)	8!
NEW 144	return null;	×
145
146	if (input.intent instanceof Ast.SpecialControlIntent) {	8✔
147	switch (input.intent.type) {	6✔
148	case 'yes':	9!
149	case 'no': {
150	const value = new Ast.BooleanValue(input.intent.type === 'yes');	6✔
151	const handled = await policy.handleAnswer(context, value);	6✔
152	if (!handled)	6!
NEW 153	return null;	×
154	return computePrediction(context, handled, 'user');	6✔
155	}
156	default:
NEW 157	return null;	×
158	}
159	}
160	if (input.intent instanceof Ast.ChoiceControlIntent)	2!
NEW 161	return null;	×
162
163	if (input.intent instanceof Ast.AnswerControlIntent) {	2!
164	const handled = await policy.handleAnswer(context, input.intent.value);	2✔
165	if (!handled)	2!
NEW 166	return null;	×
167	return computePrediction(context, handled, 'user');	2✔
168	}
169
NEW 170	throw new TypeError(`Unrecognized bookkeeping intent`);	×
171	} else if (input instanceof Ast.Program) {	64!
172	// convert thingtalk programs to dialogue states so we can use "\t" without too much typing
NEW 173	const prediction = new Ast.DialogueState(null, 'org.thingpedia.dialogue.transaction', 'execute', null, []);	×
NEW 174	for (const stmt of input.statements) {	×
NEW 175	if (stmt instanceof Ast.Assignment)	×
NEW 176	throw new Error(`Unsupported: assignment statement`);	×
NEW 177	prediction.history.push(new Ast.DialogueHistoryItem(null, stmt, null, 'accepted'));	×
178	}
NEW 179	return prediction;	×
180	}
181
182	assert(input instanceof Ast.DialogueState);	64✔
183	return input;	64✔
184	}

stanford-oval / genie-toolkit / 1660

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous