1728

pending completion

Build # 1728

Build Type

Pull #512

travis-ci-com

Committed by

web-flow

Commit Message

Merge b03e625ef into 7009cf854

Pull Request Pull Request #512: Two small fixes

Run Details

10918 of 23370 branches covered (46.72%)

Branch coverage included in aggregate %.

1 of 6 new or added lines in 1 file covered. (16.67%)

2 existing lines in 2 files now uncovered.

25492 of 44030 relevant lines covered (57.9%)

550930.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.35

/tool/interactive-annotate.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as argparse from 'argparse';
import * as fs from 'fs';
import * as readline from 'readline';
import * as events from 'events';
import seedrandom from 'seedrandom';
import * as Tp from 'thingpedia';
import * as ThingTalk from 'thingtalk';
import interpolate from 'string-interp';

import * as ThingTalkUtils from '../lib/utils/thingtalk';
import * as StreamUtils from '../lib/utils/stream-utils';
import { EntityMap } from '../lib/utils/entity-utils';
import { randint } from '../lib/utils/random';
import * as ParserClient from '../lib/prediction/parserclient';
import {
    DialogueSerializer,
    DialogueTurn,
} from '../lib/dataset-tools/parsers';
import AbstractDialogueAgent from '../lib/dialogue-agent/abstract_dialogue_agent';
import ExecutionDialogueAgent from '../lib/dialogue-agent/execution_dialogue_agent';
import DialoguePolicy from '../lib/dialogue-agent/dialogue_policy';
import ValueCategory from '../lib/dialogue-agent/value-category';
import Engine from '../lib/engine';
import * as I18n from '../lib/i18n';

import MultiJSONDatabase from './lib/multi_json_database';
import Platform from './lib/cmdline-platform';

interface AnnotatorOptions {
    locale : string;
    timezone : string|undefined;
    thingpedia_url : string;
    thingpedia_dir : string|undefined;
    nlu_server : string;
    database_file : string|undefined;
    execution_mode : 'simulation'|'real';
    random_seed : string;
}

class Annotator extends events.EventEmitter {
    private _rl : readline.Interface;
    private _locale : string;
    private _langPack : I18n.LanguagePack;
    private _rng : () => number;
    private _platform : Tp.BasePlatform;
    private _tpClient : Tp.BaseClient;
    private _schemas : ThingTalk.SchemaRetriever;
    private _parser : ParserClient.ParserClient;
    private _executor : AbstractDialogueAgent<unknown>;
    private _dialoguePolicy : DialoguePolicy;
    private _engine : Engine|undefined;
    private _simulatorDatabase : MultiJSONDatabase|undefined;

    private _state : 'loading'|'input'|'done'|'top3'|'full'|'code';
    private _serial : number;
    private _outputDialogue : DialogueTurn[];
    private _currentTurnIdx : number;
    private _outputTurn : DialogueTurn;
    private _context : ThingTalk.Ast.DialogueState|null;
    private _executorState : any|undefined;
    private _preprocessed : string|undefined;
    private _entities : EntityMap|undefined;
    private _candidates : ThingTalk.Ast.DialogueState[]|undefined;

    constructor(rl : readline.Interface, options : AnnotatorOptions) {
        super();

        this._rl = rl;

        this._locale = options.locale;
        this._langPack = I18n.get(options.locale);
        this._rng = seedrandom.alea(options.random_seed);
        this._parser = ParserClient.get(options.nlu_server, options.locale);

        this._platform = new Platform(undefined, options.locale, options.thingpedia_url);
        this._tpClient = this._platform.getCapability('thingpedia-client')!;

        if (options.execution_mode === 'simulation') {
            this._schemas = new ThingTalk.SchemaRetriever(this._tpClient, null, true);

            const simulatorOptions : ThingTalkUtils.SimulatorOptions = {
                rng: seedrandom.alea('almond is awesome'),
                locale: options.locale,
                timezone: options.timezone,
                thingpediaClient: this._tpClient,
                schemaRetriever: this._schemas,
                interactive: true
            };
            if (options.database_file) {
                this._simulatorDatabase = new MultiJSONDatabase(options.database_file);
                simulatorOptions.database = this._simulatorDatabase;
            }

            this._executor = ThingTalkUtils.createSimulator(simulatorOptions);
        } else {
            this._engine = new Engine(this._platform);
            this._executor = new ExecutionDialogueAgent(this._engine, this, false);
            this._schemas = this._engine.schemas;
        }
        this._dialoguePolicy = new DialoguePolicy({
            thingpedia: this._tpClient,
            schemas: this._schemas,
            locale: this._locale,
            timezone: options.timezone,
            rng: this._rng,
            debug: false
        });

        this._state = 'loading';

        this._serial = 0;

        this._outputDialogue = [];
        this._currentTurnIdx = 0;
        this._outputTurn = {
            context: 'null',
            agent: '',
            agent_target: '',
            intermediate_context: '',
            user: '',
            user_target: '',
        };
        this._context = null;
        this._executorState = undefined;
        this._preprocessed = undefined;
        this._entities = {};
        this._candidates = undefined;

        rl.on('line', async (line) => {
            if (this._state === 'done')
                return;

            line = line.trim();

            if (line.length === 0 || this._state === 'loading') {
                rl.prompt();
                return;
            }

            if (line === 'h' || line === '?') {
                this._help();
                return;
            }
            if (line === 'q') {
                this._quit();
                return;
            }

            if (line === 'd') {
                this.next();
                return;
            }

            if (this._state === 'input') {
                this._handleUtterance(line).catch((e) => this.emit('error', e));
                return;
            }

            if (this._state === 'code') {
                this._learnThingTalk(line).catch((e) => this.emit('error', e));
                return;
            }

            if (Number.isFinite(parseInt(line))) {
                this._learnNumber(parseInt(line));
            } else if (line === 'n') {
                this._more();
            } else if (line === 'e') {
                this._edit(undefined);
            } else if (line.startsWith('e ')) {
                this._edit(parseInt(line.substring(2).trim()));
            } else if (line === 't') {
                this._state = 'code';
                rl.setPrompt('TT: ');
                rl.prompt();
            } else {
                //console.log('Invalid command');
                //rl.prompt();
                this._learnThingTalk(line).catch((e) => this.emit('error', e));
            }
        });
    }

    // implementation of the abstract dialogue loop interface, which the
    // execution dialogue agent calls sometimes

    get _() {
        return (x : string) => x;
    }
    get icon() {
        return null;
    }
    set icon(v : string|null) {
        // do nothing
    }
    get isAnonymous() {
        return false;
    }
    get platformData() {
        return {};
    }

    interpolate(msg : string, args : Record<string, unknown>) : string {
        return interpolate(msg, args, {
            locale: this._locale,
            timezone: this._platform.timezone
        })||'';
    }

    async reply(msg : string) {
        console.log('A: ' + msg);
    }
    async replyLink(title : string, link : string) {
        console.log('A: ' + title + ' ' + link);
    }

    async replyInterp(msg : string, args ?: Record<string, unknown>) {
        if (args === undefined)
            return this.reply(msg);
        else
            return this.reply(this.interpolate(msg, args));
    }

    async ask(cat : ValueCategory, question : string) : Promise<ThingTalk.Ast.Value> {
        if (cat === ValueCategory.Location)
            return new ThingTalk.Ast.LocationValue(new ThingTalk.Ast.AbsoluteLocation(37.4299908, -122.175519, "Gates Computer Science"));
        if (cat === ValueCategory.Time)
            return new ThingTalk.Ast.TimeValue(new ThingTalk.Ast.AbsoluteTime(randint(9, 12, this._rng), 0, 0));

        throw new TypeError(`Unexpected question of type ${cat}`);
    }
    async askChoices(question : string, choices : string[]) : Promise<number> {
        return randint(0, choices.length-1, this._rng);
    }

    private _quit() {
        if (this._outputTurn.user_target)
            this._outputDialogue.push(this._outputTurn);
        if (this._outputDialogue.length > 0) {
            this.emit('learned', {
                id: String(this._serial),
                turns: this._outputDialogue,
            });
        }

        this.emit('quit');
    }

    private _help() {
        console.log('Available commands:');
        console.log('q: quit');
        console.log('d: (done) complete the current dialog and start the next one');
        console.log('<0-9>: make a choice');
        console.log('n: (next) show more choices');
        console.log('e <0-9>: edit a choice');
        console.log('t: (thingtalk) write code directly');
        console.log('? or h: this help');
    }

    async start() {
        if (this._simulatorDatabase)
            await this._simulatorDatabase.load();
        if (this._engine)
            await this._engine.open();
        await this._parser.start();
    }
    async stop() {
        await this._parser.stop();
        if (this._engine)
            await this._engine.stop();
    }

    private async _learnThingTalk(code : string) {
        let newState;
        try {
            const program = await ThingTalkUtils.parse(code, this._schemas);
            newState = await this._inputToDialogueState(program);
            if (!newState) {
                console.log(`Not a valid dialogue state`);
                this._rl.setPrompt('TT: ');
                this._rl.prompt();
                return;
            }

            // check that the entities are correct by serializing the program once
            ThingTalkUtils.serializePrediction(newState, this._preprocessed!, this._entities!, {
                locale: this._locale
            }).join(' ');
        } catch(e) {
            console.log(`${e.name}: ${e.message}`);
            this._rl.setPrompt('TT: ');
            this._rl.prompt();
            return;
        }

        const oldContext = this._context;
        this._context = ThingTalkUtils.computeNewState(this._context, newState, 'user');
        const prediction = ThingTalkUtils.computePrediction(oldContext, this._context, 'user');
        this._outputTurn.user_target = prediction.prettyprint();
        this._nextTurn();
    }

    private _edit(i : number|undefined) {
        let program;
        if (i === undefined) {
            program = this._context!;
        } else {
            if (Number.isNaN(i) || i < 1 || i > this._candidates!.length) {
                console.log('Invalid number');
                this._rl.setPrompt('$ ');
                this._rl.prompt();
                return;
            }
            i -= 1;
            program = this._candidates![i];
        }
        this._state = 'code';
        this._rl.setPrompt('TT: ');
        this._rl.write(program.prettyprint().replace(/\n/g, ' '));
        this._rl.prompt();
    }

    private _learnNumber(i : number) {
        if (i < 1 || i > this._candidates!.length) {
            console.log('Invalid number');
            this._rl.setPrompt('$ ');
            this._rl.prompt();
            return;
        }
        i -= 1;

        const program = this._candidates![i];
        const oldContext = this._context;
        this._context = ThingTalkUtils.computeNewState(this._context, program, 'user');
        const prediction = ThingTalkUtils.computePrediction(oldContext, this._context, 'user');
        this._outputTurn.user_target = prediction.prettyprint();
        this._nextTurn();
    }

    private _more() {
        if (this._state === 'top3') {
            this._state = 'full';
            const candidates = this._candidates!;
            for (let i = 0; i < candidates.length; i++)
                console.log(`${i+1}) ${candidates[i].prettyprint()}`);
            this._rl.setPrompt('$ ');
            this._rl.prompt();
        } else {
            this._state = 'code';
            this._rl.setPrompt('TT: ');
            this._rl.prompt();
        }
    }

    next() {
        if (this._outputTurn.user_target)
            this._outputDialogue.push(this._outputTurn);
        if (this._outputDialogue.length > 0) {
            this.emit('learned', {
                id: String(this._serial),
                turns: this._outputDialogue,
            });
        }

        this._serial++;
        console.log(`Dialog #${this._serial}`);

        this._outputDialogue = [];
        this._context = null;
        this._executorState = undefined;
        this._currentTurnIdx = -1;

        this._nextTurn();
    }

    private async _nextTurn() {
        if (this._outputTurn.user_target)
            this._outputDialogue.push(this._outputTurn);
        this._currentTurnIdx ++;

        this._outputTurn = {
            context: 'null',
            agent: '',
            agent_target: '',
            intermediate_context: '',
            user: '',
            user_target: '',
        };

        if (this._currentTurnIdx > 0) {
            try {
                const { newDialogueState, newExecutorState } = await this._executor.execute(this._context!, this._executorState);
                this._context = newDialogueState;
                this._executorState = newExecutorState;
            } catch(e) {
                if (e.code === 'ECANCELLED') {
                    this.next();
                    return;
                } else {
                    throw e;
                }
            }

            console.log();
            const contextCode = this._context.prettyprint();
            for (const line of contextCode.trim().split('\n'))
                console.log('C: ' + line);
            this._outputTurn.context = contextCode;

            // run the agent

            const policyResult = await this._dialoguePolicy.chooseAction(this._context);
            if (!policyResult) {
                console.log('Dialogue policy error: no reply at this state');
                this.next();
                return;
            }
            const [dialogueStateAfterAgent, , utterance, entities] = policyResult;

            const postprocessed = this._langPack.postprocessNLG(utterance, entities, this._executor);
            console.log('A: ' + postprocessed);

            const prediction = ThingTalkUtils.computePrediction(this._context, dialogueStateAfterAgent, 'agent');

            this._outputTurn.agent = postprocessed;
            this._outputTurn.agent_target = prediction.prettyprint();
            this._context = dialogueStateAfterAgent;
        }

        this._state = 'input';

        this._rl.setPrompt('U: ');
        this._rl.prompt();
    }

    private async _inputToDialogueState(input : ThingTalk.Ast.Input) : Promise<ThingTalk.Ast.DialogueState|null> {
        return ThingTalkUtils.inputToDialogueState(this._dialoguePolicy, this._context, input);
    }

    private async _handleUtterance(utterance : string) {
        this._outputTurn.user = utterance;
        this._state = 'loading';

        let contextCode, contextEntities;
        if (this._context !== null) {
            const context = ThingTalkUtils.prepareContextForPrediction(this._context, 'user');
            [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(context);
        } else {
            contextCode = ['null'];
            contextEntities = {};
        }

        const parsed = await this._parser.sendUtterance(utterance, contextCode, contextEntities, {
            tokenized: false,
            skip_typechecking: true
        });

        this._state = 'top3';
        this._preprocessed = parsed.tokens.join(' ');
        this._entities = parsed.entities;
        const candidates = await ThingTalkUtils.parseAllPredictions(parsed.candidates, parsed.entities, {
            thingpediaClient: this._tpClient,
            schemaRetriever: this._schemas,
            loadMetadata: true
        }) as ThingTalk.Ast.DialogueState[];
        this._candidates = (await Promise.all(candidates.map((c) => this._inputToDialogueState(c)))).filter((c) : c is ThingTalk.Ast.DialogueState => c !== null);

        if (this._candidates.length > 0) {
            for (let i = 0; i < 3 && i < this._candidates.length; i++)
                console.log(`${i+1}) ${this._candidates[i].prettyprint()}`);
        } else {
            console.log(`No candidates for this command`);
        }

        this._rl.setPrompt('$ ');
        this._rl.prompt();
    }
}

const THINGPEDIA_URL = 'https://dev.almond.stanford.edu/thingpedia';
const NL_SERVER_URL = 'https://nlp-staging.almond.stanford.edu';

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('interactive-annotate', {
        add_help: true,
        description: `Interactively annotate a dialog dataset, by annotating each user sentence and running the real agent.`
    });
    parser.add_argument('-o', '--output', {
        required: true,
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
    });
    parser.add_argument('--timezone', {
        required: false,
        default: undefined,
        help: `Timezone to use to print dates and times (defaults to the current timezone).`
    });
    parser.add_argument('--thingpedia-url', {
        required: false,
        default: THINGPEDIA_URL,
        help: 'URL of Thingpedia to use, or local path pointing to a thingpedia.tt file.'
    });
    parser.add_argument('--thingpedia-dir', {
        required: false,
        nargs: '+',
        help: 'Path to a directory containing Thingpedia device definitions (overrides --thingpedia-url).'
    });
    parser.add_argument('--database-file', {
        required: false,
        help: `Path to a file pointing to JSON databases used to simulate queries.`,
    });
    parser.add_argument('--nlu-server', {
        required: false,
        default: NL_SERVER_URL,
        help: `The URL of the natural language server to parse user utterances. Use a file:// URL pointing to a model directory to use a local instance of genienlp.`
    });
    parser.add_argument('--execution-mode', {
        required: false,
        default: 'real',
        choices: ['simulation', 'real'],
        help: `Whether to simulate API calls or execute them for real.`
    });
    parser.add_argument('--append', {
        action: 'store_true',
        help: 'Append to the output file instead of overwriting',
        default: false,
    });
    parser.add_argument('--no-append', {
        action: 'store_false',
        dest: 'append',
        help: 'Overwrite the output file instead of appending',
    });
}

export async function execute(args : any) {
    const learned = new DialogueSerializer({ annotations: true });
    const output = fs.createWriteStream(args.output, { flags: (args.append ? 'a' : 'w') });
    learned.pipe(output);

    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
    rl.setPrompt('$ ');

    function quit() {
        learned.end();
        rl.close();
        //process.exit();
    }

    const annotator = new Annotator(rl, args);
    await annotator.start();

    annotator.on('end', quit);
    annotator.on('learned', (dlg) => {
        learned.write(dlg);
    });
    annotator.on('quit', quit);
    rl.on('SIGINT', quit);
    annotator.next();
    //process.stdin.on('end', quit);

    await StreamUtils.waitFinish(output);
    await annotator.stop();
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as argparse from 'argparse';
22	import * as fs from 'fs';	58✔
23	import * as readline from 'readline';	58✔
24	import * as events from 'events';	58✔
25	import seedrandom from 'seedrandom';	58✔
26	import * as Tp from 'thingpedia';
27	import * as ThingTalk from 'thingtalk';	58✔
28	import interpolate from 'string-interp';	58✔
29
30	import * as ThingTalkUtils from '../lib/utils/thingtalk';	58✔
31	import * as StreamUtils from '../lib/utils/stream-utils';	58✔
32	import { EntityMap } from '../lib/utils/entity-utils';
33	import { randint } from '../lib/utils/random';	58✔
34	import * as ParserClient from '../lib/prediction/parserclient';	58✔
35	import {	58✔
36	DialogueSerializer,
37	DialogueTurn,
38	} from '../lib/dataset-tools/parsers';
39	import AbstractDialogueAgent from '../lib/dialogue-agent/abstract_dialogue_agent';
40	import ExecutionDialogueAgent from '../lib/dialogue-agent/execution_dialogue_agent';	58✔
41	import DialoguePolicy from '../lib/dialogue-agent/dialogue_policy';	58✔
42	import ValueCategory from '../lib/dialogue-agent/value-category';	58✔
43	import Engine from '../lib/engine';	58✔
44	import * as I18n from '../lib/i18n';	58✔
45
46	import MultiJSONDatabase from './lib/multi_json_database';	58✔
47	import Platform from './lib/cmdline-platform';	58✔
48
49	interface AnnotatorOptions {
50	locale : string;
51	timezone : string\|undefined;
52	thingpedia_url : string;
53	thingpedia_dir : string\|undefined;
54	nlu_server : string;
55	database_file : string\|undefined;
56	execution_mode : 'simulation'\|'real';
57	random_seed : string;
58	}
59
60	class Annotator extends events.EventEmitter {
61	private _rl : readline.Interface;
62	private _locale : string;
63	private _langPack : I18n.LanguagePack;
64	private _rng : () => number;
65	private _platform : Tp.BasePlatform;
66	private _tpClient : Tp.BaseClient;
67	private _schemas : ThingTalk.SchemaRetriever;
68	private _parser : ParserClient.ParserClient;
69	private _executor : AbstractDialogueAgent<unknown>;
70	private _dialoguePolicy : DialoguePolicy;
71	private _engine : Engine\|undefined;
72	private _simulatorDatabase : MultiJSONDatabase\|undefined;
73
74	private _state : 'loading'\|'input'\|'done'\|'top3'\|'full'\|'code';
75	private _serial : number;
76	private _outputDialogue : DialogueTurn[];
77	private _currentTurnIdx : number;
78	private _outputTurn : DialogueTurn;
79	private _context : ThingTalk.Ast.DialogueState\|null;
80	private _executorState : any\|undefined;
81	private _preprocessed : string\|undefined;
82	private _entities : EntityMap\|undefined;
83	private _candidates : ThingTalk.Ast.DialogueState[]\|undefined;
84
85	constructor(rl : readline.Interface, options : AnnotatorOptions) {
86	super();	×
87
88	this._rl = rl;	×
89
90	this._locale = options.locale;	×
NEW 91	this._langPack = I18n.get(options.locale);	×
92	this._rng = seedrandom.alea(options.random_seed);	×
93	this._parser = ParserClient.get(options.nlu_server, options.locale);	×
94
95	this._platform = new Platform(undefined, options.locale, options.thingpedia_url);	×
96	this._tpClient = this._platform.getCapability('thingpedia-client')!;	×
97
98	if (options.execution_mode === 'simulation') {	×
99	this._schemas = new ThingTalk.SchemaRetriever(this._tpClient, null, true);	×
100
101	const simulatorOptions : ThingTalkUtils.SimulatorOptions = {	×
102	rng: seedrandom.alea('almond is awesome'),
103	locale: options.locale,
104	timezone: options.timezone,
105	thingpediaClient: this._tpClient,
106	schemaRetriever: this._schemas,
107	interactive: true
108	};
109	if (options.database_file) {	×
110	this._simulatorDatabase = new MultiJSONDatabase(options.database_file);	×
111	simulatorOptions.database = this._simulatorDatabase;	×
112	}
113
114	this._executor = ThingTalkUtils.createSimulator(simulatorOptions);	×
115	} else {
116	this._engine = new Engine(this._platform);	×
117	this._executor = new ExecutionDialogueAgent(this._engine, this, false);	×
118	this._schemas = this._engine.schemas;	×
119	}
120	this._dialoguePolicy = new DialoguePolicy({	×
121	thingpedia: this._tpClient,
122	schemas: this._schemas,
123	locale: this._locale,
124	timezone: options.timezone,
125	rng: this._rng,
126	debug: false
127	});
128
129	this._state = 'loading';	×
130
131	this._serial = 0;	×
132
133	this._outputDialogue = [];	×
134	this._currentTurnIdx = 0;	×
135	this._outputTurn = {	×
136	context: 'null',
137	agent: '',
138	agent_target: '',
139	intermediate_context: '',
140	user: '',
141	user_target: '',
142	};
143	this._context = null;	×
144	this._executorState = undefined;	×
145	this._preprocessed = undefined;	×
146	this._entities = {};	×
147	this._candidates = undefined;	×
148
149	rl.on('line', async (line) => {	×
150	if (this._state === 'done')	×
151	return;	×
152
153	line = line.trim();	×
154
155	if (line.length === 0 \|\| this._state === 'loading') {	×
156	rl.prompt();	×
157	return;	×
158	}
159
160	if (line === 'h' \|\| line === '?') {	×
161	this._help();	×
162	return;	×
163	}
164	if (line === 'q') {	×
165	this._quit();	×
166	return;	×
167	}
168
169	if (line === 'd') {	×
170	this.next();	×
171	return;	×
172	}
173
174	if (this._state === 'input') {	×
175	this._handleUtterance(line).catch((e) => this.emit('error', e));	×
176	return;	×
177	}
178
179	if (this._state === 'code') {	×
180	this._learnThingTalk(line).catch((e) => this.emit('error', e));	×
181	return;	×
182	}
183
184	if (Number.isFinite(parseInt(line))) {	×
185	this._learnNumber(parseInt(line));	×
186	} else if (line === 'n') {	×
187	this._more();	×
188	} else if (line === 'e') {	×
189	this._edit(undefined);	×
190	} else if (line.startsWith('e ')) {	×
191	this._edit(parseInt(line.substring(2).trim()));	×
192	} else if (line === 't') {	×
193	this._state = 'code';	×
194	rl.setPrompt('TT: ');	×
195	rl.prompt();	×
196	} else {
197	//console.log('Invalid command');
198	//rl.prompt();
199	this._learnThingTalk(line).catch((e) => this.emit('error', e));	×
200	}
201	});
202	}
203
204	// implementation of the abstract dialogue loop interface, which the
205	// execution dialogue agent calls sometimes
206
207	get _() {
208	return (x : string) => x;	×
209	}
210	get icon() {
211	return null;	×
212	}
213	set icon(v : string\|null) {
214	// do nothing
215	}
216	get isAnonymous() {
217	return false;	×
218	}
219	get platformData() {
220	return {};	×
221	}
222
223	interpolate(msg : string, args : Record<string, unknown>) : string {
224	return interpolate(msg, args, {	×
225	locale: this._locale,
226	timezone: this._platform.timezone
227	})\|\|'';
228	}
229
230	async reply(msg : string) {
231	console.log('A: ' + msg);	×
232	}
233	async replyLink(title : string, link : string) {
234	console.log('A: ' + title + ' ' + link);	×
235	}
236
237	async replyInterp(msg : string, args ?: Record<string, unknown>) {
238	if (args === undefined)	×
239	return this.reply(msg);	×
240	else
241	return this.reply(this.interpolate(msg, args));	×
242	}
243
244	async ask(cat : ValueCategory, question : string) : Promise<ThingTalk.Ast.Value> {
245	if (cat === ValueCategory.Location)	×
246	return new ThingTalk.Ast.LocationValue(new ThingTalk.Ast.AbsoluteLocation(37.4299908, -122.175519, "Gates Computer Science"));	×
247	if (cat === ValueCategory.Time)	×
248	return new ThingTalk.Ast.TimeValue(new ThingTalk.Ast.AbsoluteTime(randint(9, 12, this._rng), 0, 0));	×
249
250	throw new TypeError(`Unexpected question of type ${cat}`);	×
251	}
252	async askChoices(question : string, choices : string[]) : Promise<number> {
253	return randint(0, choices.length-1, this._rng);	×
254	}
255
256	private _quit() {
257	if (this._outputTurn.user_target)	×
258	this._outputDialogue.push(this._outputTurn);	×
259	if (this._outputDialogue.length > 0) {	×
260	this.emit('learned', {	×
261	id: String(this._serial),
262	turns: this._outputDialogue,
263	});
264	}
265
266	this.emit('quit');	×
267	}
268
269	private _help() {
270	console.log('Available commands:');	×
271	console.log('q: quit');	×
272	console.log('d: (done) complete the current dialog and start the next one');	×
273	console.log('<0-9>: make a choice');	×
274	console.log('n: (next) show more choices');	×
275	console.log('e <0-9>: edit a choice');	×
276	console.log('t: (thingtalk) write code directly');	×
277	console.log('? or h: this help');	×
278	}
279
280	async start() {
281	if (this._simulatorDatabase)	×
282	await this._simulatorDatabase.load();	×
283	if (this._engine)	×
284	await this._engine.open();	×
285	await this._parser.start();	×
286	}
287	async stop() {
288	await this._parser.stop();	×
289	if (this._engine)	×
290	await this._engine.stop();	×
291	}
292
293	private async _learnThingTalk(code : string) {
294	let newState;
295	try {	×
296	const program = await ThingTalkUtils.parse(code, this._schemas);	×
297	newState = await this._inputToDialogueState(program);	×
298	if (!newState) {	×
299	console.log(`Not a valid dialogue state`);	×
300	this._rl.setPrompt('TT: ');	×
301	this._rl.prompt();	×
302	return;	×
303	}
304
305	// check that the entities are correct by serializing the program once
306	ThingTalkUtils.serializePrediction(newState, this._preprocessed!, this._entities!, {	×
307	locale: this._locale
308	}).join(' ');
309	} catch(e) {
310	console.log(`${e.name}: ${e.message}`);	×
311	this._rl.setPrompt('TT: ');	×
312	this._rl.prompt();	×
313	return;	×
314	}
315
316	const oldContext = this._context;	×
317	this._context = ThingTalkUtils.computeNewState(this._context, newState, 'user');	×
318	const prediction = ThingTalkUtils.computePrediction(oldContext, this._context, 'user');	×
319	this._outputTurn.user_target = prediction.prettyprint();	×
320	this._nextTurn();	×
321	}
322
323	private _edit(i : number\|undefined) {
324	let program;
325	if (i === undefined) {	×
326	program = this._context!;	×
327	} else {
328	if (Number.isNaN(i) \|\| i < 1 \|\| i > this._candidates!.length) {	×
329	console.log('Invalid number');	×
330	this._rl.setPrompt('$ ');	×
331	this._rl.prompt();	×
332	return;	×
333	}
334	i -= 1;	×
335	program = this._candidates![i];	×
336	}
337	this._state = 'code';	×
338	this._rl.setPrompt('TT: ');	×
339	this._rl.write(program.prettyprint().replace(/\n/g, ' '));	×
340	this._rl.prompt();	×
341	}
342
343	private _learnNumber(i : number) {
344	if (i < 1 \|\| i > this._candidates!.length) {	×
345	console.log('Invalid number');	×
346	this._rl.setPrompt('$ ');	×
347	this._rl.prompt();	×
348	return;	×
349	}
350	i -= 1;	×
351
352	const program = this._candidates![i];	×
353	const oldContext = this._context;	×
354	this._context = ThingTalkUtils.computeNewState(this._context, program, 'user');	×
355	const prediction = ThingTalkUtils.computePrediction(oldContext, this._context, 'user');	×
356	this._outputTurn.user_target = prediction.prettyprint();	×
357	this._nextTurn();	×
358	}
359
360	private _more() {
361	if (this._state === 'top3') {	×
362	this._state = 'full';	×
363	const candidates = this._candidates!;	×
364	for (let i = 0; i < candidates.length; i++)	×
365	console.log(`${i+1}) ${candidates[i].prettyprint()}`);	×
366	this._rl.setPrompt('$ ');	×
367	this._rl.prompt();	×
368	} else {
369	this._state = 'code';	×
370	this._rl.setPrompt('TT: ');	×
371	this._rl.prompt();	×
372	}
373	}
374
375	next() {
376	if (this._outputTurn.user_target)	×
377	this._outputDialogue.push(this._outputTurn);	×
378	if (this._outputDialogue.length > 0) {	×
379	this.emit('learned', {	×
380	id: String(this._serial),
381	turns: this._outputDialogue,
382	});
383	}
384
385	this._serial++;	×
386	console.log(`Dialog #${this._serial}`);	×
387
388	this._outputDialogue = [];	×
389	this._context = null;	×
390	this._executorState = undefined;	×
391	this._currentTurnIdx = -1;	×
392
393	this._nextTurn();	×
394	}
395
396	private async _nextTurn() {
397	if (this._outputTurn.user_target)	×
398	this._outputDialogue.push(this._outputTurn);	×
399	this._currentTurnIdx ++;	×
400
401	this._outputTurn = {	×
402	context: 'null',
403	agent: '',
404	agent_target: '',
405	intermediate_context: '',
406	user: '',
407	user_target: '',
408	};
409
410	if (this._currentTurnIdx > 0) {	×
411	try {	×
412	const { newDialogueState, newExecutorState } = await this._executor.execute(this._context!, this._executorState);	×
413	this._context = newDialogueState;	×
414	this._executorState = newExecutorState;	×
415	} catch(e) {
416	if (e.code === 'ECANCELLED') {	×
417	this.next();	×
418	return;	×
419	} else {
420	throw e;	×
421	}
422	}
423
424	console.log();	×
425	const contextCode = this._context.prettyprint();	×
426	for (const line of contextCode.trim().split('\n'))	×
427	console.log('C: ' + line);	×
428	this._outputTurn.context = contextCode;	×
429
430	// run the agent
431
432	const policyResult = await this._dialoguePolicy.chooseAction(this._context);	×
433	if (!policyResult) {	×
434	console.log('Dialogue policy error: no reply at this state');	×
435	this.next();	×
436	return;	×
437	}
NEW 438	const [dialogueStateAfterAgent, , utterance, entities] = policyResult;	×
439
NEW 440	const postprocessed = this._langPack.postprocessNLG(utterance, entities, this._executor);	×
NEW 441	console.log('A: ' + postprocessed);	×
442
443	const prediction = ThingTalkUtils.computePrediction(this._context, dialogueStateAfterAgent, 'agent');	×
444
NEW 445	this._outputTurn.agent = postprocessed;	×
446	this._outputTurn.agent_target = prediction.prettyprint();	×
447	this._context = dialogueStateAfterAgent;	×
448	}
449
450	this._state = 'input';	×
451
452	this._rl.setPrompt('U: ');	×
453	this._rl.prompt();	×
454	}
455
456	private async _inputToDialogueState(input : ThingTalk.Ast.Input) : Promise<ThingTalk.Ast.DialogueState\|null> {
457	return ThingTalkUtils.inputToDialogueState(this._dialoguePolicy, this._context, input);	×
458	}
459
460	private async _handleUtterance(utterance : string) {
461	this._outputTurn.user = utterance;	×
462	this._state = 'loading';	×
463
464	let contextCode, contextEntities;
465	if (this._context !== null) {	×
466	const context = ThingTalkUtils.prepareContextForPrediction(this._context, 'user');	×
467	[contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(context);	×
468	} else {
469	contextCode = ['null'];	×
470	contextEntities = {};	×
471	}
472
473	const parsed = await this._parser.sendUtterance(utterance, contextCode, contextEntities, {	×
474	tokenized: false,
475	skip_typechecking: true
476	});
477
478	this._state = 'top3';	×
479	this._preprocessed = parsed.tokens.join(' ');	×
480	this._entities = parsed.entities;	×
481	const candidates = await ThingTalkUtils.parseAllPredictions(parsed.candidates, parsed.entities, {	×
482	thingpediaClient: this._tpClient,
483	schemaRetriever: this._schemas,
484	loadMetadata: true
485	}) as ThingTalk.Ast.DialogueState[];
486	this._candidates = (await Promise.all(candidates.map((c) => this._inputToDialogueState(c)))).filter((c) : c is ThingTalk.Ast.DialogueState => c !== null);	×
487
488	if (this._candidates.length > 0) {	×
489	for (let i = 0; i < 3 && i < this._candidates.length; i++)	×
490	console.log(`${i+1}) ${this._candidates[i].prettyprint()}`);	×
491	} else {
492	console.log(`No candidates for this command`);	×
493	}
494
495	this._rl.setPrompt('$ ');	×
496	this._rl.prompt();	×
497	}
498	}
499
500	const THINGPEDIA_URL = 'https://dev.almond.stanford.edu/thingpedia';	58✔
501	const NL_SERVER_URL = 'https://nlp-staging.almond.stanford.edu';	58✔
502
503	export function initArgparse(subparsers : argparse.SubParser) {	58✔
504	const parser = subparsers.add_parser('interactive-annotate', {	58✔
505	add_help: true,
506	description: `Interactively annotate a dialog dataset, by annotating each user sentence and running the real agent.`
507	});
508	parser.add_argument('-o', '--output', {	58✔
509	required: true,
510	});
511	parser.add_argument('-l', '--locale', {	58✔
512	required: false,
513	default: 'en-US',
514	help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
515	});
516	parser.add_argument('--timezone', {	58✔
517	required: false,
518	default: undefined,
519	help: `Timezone to use to print dates and times (defaults to the current timezone).`
520	});
521	parser.add_argument('--thingpedia-url', {	58✔
522	required: false,
523	default: THINGPEDIA_URL,
524	help: 'URL of Thingpedia to use, or local path pointing to a thingpedia.tt file.'
525	});
526	parser.add_argument('--thingpedia-dir', {	58✔
527	required: false,
528	nargs: '+',
529	help: 'Path to a directory containing Thingpedia device definitions (overrides --thingpedia-url).'
530	});
531	parser.add_argument('--database-file', {	58✔
532	required: false,
533	help: `Path to a file pointing to JSON databases used to simulate queries.`,
534	});
535	parser.add_argument('--nlu-server', {	58✔
536	required: false,
537	default: NL_SERVER_URL,
538	help: `The URL of the natural language server to parse user utterances. Use a file:// URL pointing to a model directory to use a local instance of genienlp.`
539	});
540	parser.add_argument('--execution-mode', {	58✔
541	required: false,
542	default: 'real',
543	choices: ['simulation', 'real'],
544	help: `Whether to simulate API calls or execute them for real.`
545	});
546	parser.add_argument('--append', {	58✔
547	action: 'store_true',
548	help: 'Append to the output file instead of overwriting',
549	default: false,
550	});
551	parser.add_argument('--no-append', {	58✔
552	action: 'store_false',
553	dest: 'append',
554	help: 'Overwrite the output file instead of appending',
555	});
556	}
557
558	export async function execute(args : any) {	58✔
559	const learned = new DialogueSerializer({ annotations: true });	×
560	const output = fs.createWriteStream(args.output, { flags: (args.append ? 'a' : 'w') });	×
561	learned.pipe(output);	×
562
563	const rl = readline.createInterface({ input: process.stdin, output: process.stdout });	×
564	rl.setPrompt('$ ');	×
565
566	function quit() {
567	learned.end();	×
568	rl.close();	×
569	//process.exit();
570	}
571
572	const annotator = new Annotator(rl, args);	×
573	await annotator.start();	×
574
575	annotator.on('end', quit);	×
576	annotator.on('learned', (dlg) => {	×
577	learned.write(dlg);	×
578	});
579	annotator.on('quit', quit);	×
580	rl.on('SIGINT', quit);	×
581	annotator.next();	×
582	//process.stdin.on('end', quit);
583
584	await StreamUtils.waitFinish(output);	×
585	await annotator.stop();	×
586	}

stanford-oval / genie-toolkit / 1728

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous