2508

pending completion

Build # 2508

Build Type

Pull #825

travis-ci-com

Committed by

web-flow

Commit Message

Merge c449e4c01 into e3b956225

Pull Request Pull Request #825: Fixes for internationalization

Run Details

10511 of 21165 branches covered (49.66%)

Branch coverage included in aggregate %.

149 of 164 new or added lines in 9 files covered. (90.85%)

13 existing lines in 5 files now uncovered.

25137 of 41852 relevant lines covered (60.06%)

338720.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.03

/lib/dialogue-agent/abstract_dialogue_agent.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import assert from 'assert';
import * as Tp from 'thingpedia';
import { Ast, SchemaRetriever, Builtin } from 'thingtalk';

import * as I18n from '../i18n';
import { cleanKind } from '../utils/misc-utils';
import { shouldAutoConfirmStatement, addIndexToIDQuery } from '../utils/thingtalk';
import { contactSearch, Contact } from './entity-linking/contact_search';
import { collectDisambiguationHints, getBestEntityMatch, EntityRecord } from './entity-linking/entity-finder';

import { CancellationError } from './errors';
import ValueCategory from './value-category';

interface AbstractDialogueAgentOptions {
    locale : string;
    timezone : string;
    debug : boolean;
}

export interface NewProgramRecord {
    uniqueId : string;
    name : string;
    code : string;
    results : Array<Record<string, unknown>>;
    errors : string[];
    icon : string|null;
}

export interface NotificationConfig {
    backend : string;
    config : Record<string, string>;
}

export type RawExecutionResult = Array<[string, Record<string, unknown>]>;
interface AbstractStatementExecutor<PrivateStateType> {
    executeStatement(stmt : Ast.ExpressionStatement, privateState : PrivateStateType|undefined, notificationConfig : NotificationConfig|undefined) :
        Promise<[Ast.DialogueHistoryResultList, RawExecutionResult, NewProgramRecord|undefined, PrivateStateType, Ast.AnnotationSpec]>;
}

export interface DisambiguationHints {
    devices : Map<string, [string|null, Ast.InputParam|undefined]>;
    idEntities : Map<string, EntityRecord[]>;
    previousLocations : Ast.AbsoluteLocation[];
}

export interface DeviceInfo {
    kind : string;
    uniqueId : string;
    name : string;
}


interface ExecutionResult<PrivateStateType> {
    newDialogueState : Ast.DialogueState;
    newExecutorState : PrivateStateType|undefined;
    newResults : Array<[string, Record<string, unknown>]>;
    newPrograms : NewProgramRecord[];
    anyChange : boolean;
}

/**
 * Base class of a dialogue agent.
 *
 * The class contains the code that is common between execution (aka inference time)
 * and simulation (aka training time).
 * Because it is used for simulation, the class is completely stateless
 *
 * The public API of this class is used by the entity linking & device choice code
 * (during the "prepare for execution" step) to abstract the access to the "user".
 *
 * There are two subclasses, one used during simulation, and one used for execution.
 */
export default abstract class AbstractDialogueAgent<PrivateStateType> {
    protected _tpClient : Tp.BaseClient;
    protected _schemas : SchemaRetriever;
    protected _debug : boolean;
    private _langPack : I18n.LanguagePack;
    locale : string;
    timezone : string;

    constructor(tpClient : Tp.BaseClient, schemas : SchemaRetriever, options : AbstractDialogueAgentOptions) {
        this._tpClient = tpClient;
        this._schemas = schemas;
        this._langPack = I18n.get(options.locale);

        this._debug = options.debug;
        this.locale = options.locale;
        this.timezone = options.timezone;
    }

    /**
     * Execute the query or action implied by the current dialogue state.
     *
     * This method should return a new dialogue state with filled information
     * about the result. It should not modify the state in-place.
     *
     * This is the only public method in this class.
     *
     * @param {Ast.DialogueState} state - the current state, representing the query or action to execute
     * @param {any} privateState - additional state carried by the dialogue agent (per dialogue)
     * @return {Ast.DialogueState} - the new state, with information about the returned query or action
     */
    async execute(state : Ast.DialogueState, privateState : PrivateStateType|undefined) : Promise<ExecutionResult<PrivateStateType>> {
        let cloned = false;
        let clone = state;
        let anyChange = false;

        const newResults : RawExecutionResult = [];
        const newPrograms : NewProgramRecord[] = [];
        const hints = this._collectDisambiguationHintsForState(state);
        for (let i = 0; i < clone.history.length; i++) {
            if (clone.history[i].results !== null)
                continue;
            if (clone.history[i].confirm === 'proposed')
                continue;

            if (!cloned) {
                clone = state.clone();
                cloned = true;
            }
            // prepare for execution now, even if we don't execute yet
            // so we slot-fill eagerly
            const item = clone.history[i];
            await this._prepareForExecution(item.stmt, hints);

            // if we did not execute the previous item we're not executing this one either
            if (i > 0 && clone.history[i-1].results === null)
                continue;
            if (item.confirm === 'accepted' &&
                item.isExecutable() &&
                shouldAutoConfirmStatement(item.stmt))
                item.confirm = 'confirmed';
            if (item.confirm !== 'confirmed')
                continue;
            anyChange = true;
            assert(item.isExecutable());

            // if we have a stream, we'll trigger notifications
            // configure them if necessary
            // we do this last after slot-filling and confirmations,
            // because we use the configuration immediately
            let notificationConfig = undefined;
            if (item.stmt.stream)
                notificationConfig = await this.configureNotifications();

            const [newResultList, newRawResult, newProgram, newPrivateState, annotations] = await this.executor.executeStatement(item.stmt, privateState, notificationConfig);
            item.results = newResultList;
            Object.assign(item.nl_annotations, annotations.nl ?? {});
            Object.assign(item.impl_annotations, annotations.impl ?? {});
            newResults.push(...newRawResult);
            if (newProgram)
                newPrograms.push(newProgram);
            privateState = newPrivateState;
        }

        return { newDialogueState: clone, newExecutorState: privateState, newResults, newPrograms, anyChange };
    }

    private _collectDisambiguationHintsForState(state : Ast.DialogueState) {
        const idEntities = new Map<string, EntityRecord[]>();
        const devices = new Map<string, [string|null, Ast.InputParam|undefined]>();
        const previousLocations : Ast.AbsoluteLocation[] = [];

        // collect all ID entities and all locations from the state
        for (const item of state.history) {
            const results = item.results;
            if (results === null)
                continue;

            for (const slot of item.stmt.iterateSlots2()) {
                if (slot instanceof Ast.DeviceSelector)
                    devices.set(slot.kind, [slot.id, slot.getAttribute('name')]);
            }

            for (const result of results.results)
                collectDisambiguationHints(result, idEntities, previousLocations);
        }

        return { devices, idEntities, previousLocations };
    }

    /**
     * Ensure that notifications are configured for the user, and gather
     * all the information we need.
     */
    protected abstract configureNotifications() : Promise<NotificationConfig|undefined>;

    /**
     * Check that a given statement is permitted for the current user.
     *
     * This is the hook to prevent running unsafe statements in anonymous mode.
     */
    protected abstract checkForPermission(stmt : Ast.ExpressionStatement) : Promise<void>;

    /**
     * Prepare a statement for being executed.
     *
     * This will resolve all ThingTalk values that are relative to the user, such as
     * `$context.location.current_location`, and will assign device IDs to all Thingpedia
     * function calls.
     *
     * The statement is modified in place.
     *
     * @param {thingtalk.Ast.Statement} stmt - the statement to prepare
     * @param {any} hints - hints to use to resolve any ambiguity
     */
    protected async _prepareForExecution(stmt : Ast.ExpressionStatement, hints : DisambiguationHints) : Promise<void> {
        // somewhat of a HACK:
        // add a [1] clause to immediate-mode statements that refer to "id" in
        // the query and don't have an index clause already
        addIndexToIDQuery(stmt);

        await this.checkForPermission(stmt);

        // FIXME this method can cause a few questions that
        // bypass the neural network, which is not great
        //
        // In particular, the following questions need to be fixed:
        // - "Who do you want to contact?" (from contact_search)
        // - "Where are you now?" / "What is your home address?" ... (from user_context)
        //
        // OTOH, questions that use "askChoices" or raw mode are fine,
        // because those are intentionally skipping the neural network
        //
        // (OTOOH, a proper IoT skill would probably use a more conversational
        // model for device choice, so the user can say "use the light in this room"
        // or "use the light closest to me", or similar)

        stmt.visit(new class extends Ast.NodeVisitor {
            visitInvocation(invocation : Ast.Invocation) {
                const set = new Set;
                for (const inParam of invocation.in_params)
                    set.add(inParam.name);

                for (const arg of invocation.schema!.iterateArguments()) {
                    if (arg.is_input && !arg.required && arg.impl_annotations.default && !set.has(arg.name))
                        invocation.in_params.push(new Ast.InputParam(null, arg.name, arg.impl_annotations.default));
                }
                return false;
            }
        });

        for (const slot of stmt.iterateSlots2()) {
            if (slot instanceof Ast.DeviceSelector)
                await this._chooseDevice(slot, hints);
            else
                await this._concretizeValue(slot, hints);
        }
    }

    /**
     * Show a debug message.
     *
     * @param {any} msg - what to show
     */
    debug(...args : unknown[]) : void {
        if (!this._debug)
            return;
        console.log(...args);
    }

    private async _chooseDevice(selector : Ast.DeviceSelector, hints : DisambiguationHints) : Promise<void> {
        if (selector.id !== null)
            return;

        const kind = selector.kind;
        const name = selector.getAttribute('name');
        if (!name && hints.devices.has(kind) && !selector.all) {
            // if we have already selected a device for this kind in the context, reuse what
            // we chose before without asking again
            const [previousId, previousName] = hints.devices.get(kind)!;
            selector.id = previousId;
            if (previousName)
                selector.attributes.push(previousName.clone());
            return;
        }

        const alldevices = await this.getAllDevicesOfKind(kind);

        if (alldevices.length === 0) {
            this.debug('No device of kind ' + kind + ' available, attempting configure...');
            const device = await this.tryConfigureDevice(kind);
            if (!device) // cancel the dialogue if we failed to set up a device
                throw new CancellationError();
            if (selector.all)
                return;
            selector.id = device.uniqueId;
            if (name)
                name.value = new Ast.Value.String(device.name);
            else
                selector.attributes.push(new Ast.InputParam(null, 'name', new Ast.Value.String(device.name)));
            return;
        }

        if (selector.all)
            return;

        let selecteddevices = alldevices;
        // note: we ignore the name if there is only one device configured - this protects against some bad parses
        if (alldevices.length > 1 && name !== undefined)
            selecteddevices = alldevices.filter((d) => Builtin.like(d.name, name.value.toJS() as string));

        if (selecteddevices.length === 1) {
            selector.id = selecteddevices[0].uniqueId;
            if (name)
                name.value = new Ast.Value.String(selecteddevices[0].name);
            else
                selector.attributes.push(new Ast.InputParam(null, 'name', new Ast.Value.String(selecteddevices[0].name)));
            return;
        }

        const choosefrom = (selecteddevices.length ? selecteddevices : alldevices);
        const choice = await this.disambiguate(selecteddevices.length ? 'device' : 'device-missing',
            name ? name.value.toJS() as string : null, choosefrom.map((d) => d.name), kind);
        selector.id = choosefrom[choice].uniqueId;
        if (name)
            name.value = new Ast.Value.String(choosefrom[choice].name);
        else
            selector.attributes.push(new Ast.InputParam(null, 'name', new Ast.Value.String(choosefrom[choice].name)));
    }

    private async _addDisplayToDevice(value : Ast.EntityValue) : Promise<void> {
        try {
            const classDef = await this._schemas.getFullMeta(value.value!);

            value.display = classDef.metadata.thingpedia_name || classDef.metadata.canonical || cleanKind(value.value!);
        } catch(e) {
            /* ignore if the device does not exist, it might be a constant of the form
               "str:ENTITY_tt:device::" */
        }
    }

    private async _maybeAddDisplayToValue(value : Ast.EntityValue) : Promise<void> {
        switch (value.type) {
        case 'tt:contact':
            await this.addDisplayToContact(value);
            break;

        case 'tt:device':
            await this._addDisplayToDevice(value);
            break;
        }
    }

    private async _concretizeValue(slot : Ast.AbstractSlot, hints : DisambiguationHints) : Promise<void> {
        let value = slot.get();

        // default units (e.g. defaultTemperature) will be concretized
        // according to the user's preferences or locale
        // since dlg.locale is overwritten to be en-US, we infer the locale
        // via other environment variables like LANG (language) or TZ (timezone)
        if (value instanceof Ast.MeasureValue && value.unit.startsWith('default')) {
            const key = value.unit.substring('default'.length).toLowerCase();
            const preference = this.getPreferredUnit(key);
            if (preference) {
                value.unit = preference;
            } else {
                switch (key) {
                case 'temperature':
                    value.unit = this._langPack.getDefaultTemperatureUnit();
                    break;
                default:
                    throw new TypeError('Unexpected default unit ' + value.unit);
                }
            }
        } else if (value instanceof Ast.LocationValue && value.value instanceof Ast.UnresolvedLocation) {
            slot.set(await this.lookupLocation(value.value.name, hints.previousLocations || []));
        } else if (value instanceof Ast.LocationValue && value.value instanceof Ast.RelativeLocation) {
            slot.set(await this.resolveUserContext('$context.location.' + value.value.relativeTag));
        } else if (value instanceof Ast.TimeValue && value.value instanceof Ast.RelativeTime) {
            slot.set(await this.resolveUserContext('$context.time.' + value.value.relativeTag));
        } else if (value instanceof Ast.EntityValue && value.value === null) {
            if (value.type === 'tt:email_address' || value.type === 'tt:phone_number' ||
                value.type === 'tt:contact') {
                slot.set(await contactSearch(this, value.type, value.display!));
            } else if (value.type === 'tt:device') {
                const candidates = await this._tpClient.searchDevice(value.display!);
                const tokenizer = this._langPack.getTokenizer();
                const resolved = getBestEntityMatch(value.display!, value.type, candidates.map((d) => ({
                    value: d.primary_kind,
                    name: d.name,
                    canonical: tokenizer.tokenize(d.name).rawTokens.join(' ')
                })));
                value.value = resolved.value;
                value.display = resolved.name;
            } else {
                const resolved = await this.resolveEntity(value.type, value.display!, hints);
                value.value = resolved.value;
                value.display = resolved.name;
            }
        } else if (value instanceof Ast.DateValue && value.value instanceof Ast.WeekDayDate &&
            slot.primitive?.schema?.qualifiedName === '.ontimer' && /^in_param\.date\.[0-9]+$/.test(slot.tag) &&
            value.value.time === null) {
            const time = await this.resolveUserContext('$context.time.morning');
            assert(time instanceof Ast.TimeValue && time.value instanceof Ast.AbsoluteTime);
            value.value.time = time.value;
        }

        value = slot.get();
        assert(value.isConcrete());
        if (value instanceof Ast.EntityValue && !value.display)
            await this._maybeAddDisplayToValue(value);
    }

    // The rest is the API that subclasses must implement

    /* instanbul ignore next */
    /**
     * Retrieve the executor to use for each statement.
     */
    protected get executor() : AbstractStatementExecutor<PrivateStateType> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * List all configured devices that implement the given ThingTalk kind.
     *
     * @param {string} kind - the kind to check
     * @returns {Array<DeviceInfo>} - the list of configured devices
     */
    protected async getAllDevicesOfKind(kind : string) : Promise<DeviceInfo[]> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Attempt to automatically configure a device of the given kind.
     *
     * @param {string} kind - the kind to configure
     * @returns {DeviceInfo} - the newly configured device
     */
    protected async tryConfigureDevice(kind : string) : Promise<DeviceInfo|null> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Disambiguate an entity by explicitly asking the user.
     *
     * @param {string} type - the type of disambiguation to perform, either `device`, `contact`,
     *   or `entity`
     * @param {string} name - the name to disambiguate
     * @param {string[]} choices - the choices among which to disambiguate
     * @param {string} hint - a type-specific hint to show to the user
     * @returns {number} - the index of the provided choice
     */
    async disambiguate(type : 'device'|'device-missing'|'contact',
                       name : string|null,
                       choices : string[],
                       hint ?: string) : Promise<number> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Lookup a contact in the address book.
     *
     * @param {ValueCategory} category - the category of information to look up
     * @param {string} name - the name to look up
     * @returns {string[]} - the list of resolved information of all contacts matching the name
     */
    async lookupContact(category : ValueCategory, name : string) : Promise<Contact[]> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Add the display field to a phone or email entity, by looking up the contact in the address book.
     *
     * @param {thingtalk.Ast.Value} contact - the entity to look up
     */
    protected async addDisplayToContact(contact : Ast.EntityValue) : Promise<void> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Ask the user about a contact that is not in the address book.
     *
     * @param {ValueCategory} category - the category of information to look up
     * @param {string} name - the name to look up
     * @returns {thingtalk.Ast.Value.Entity} - the entity corresponding to the picked up information
     */
    async askMissingContact(category : ValueCategory.EmailAddress|ValueCategory.PhoneNumber|ValueCategory.Contact,
                            name : string) : Promise<Ast.EntityValue> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Resolve a location name to a specific point on Earth.
     *
     * @param {string} searchKey - the location name to look up
     * @param {thingtalk.Ast.Location[]} previousLocations - recently mentioned locations
     * @returns {thingtalk.Ast.Value} - the best match for the given name
     */
    protected async lookupLocation(searchKey : string, previousLocations : Ast.AbsoluteLocation[]) : Promise<Ast.LocationValue> {
        throw new TypeError('Abstract method');
    }

    /**
     * Resolve an entity using the Thingpedia database or by calling the underlying API.
     *
     * @param entityType - the type of entity to look up
     * @param entityDisplay - the display name of the entity look up
     * @param hints - hints to use to resolve any ambiguity
     * @returns - the entity that matched the given name, in Thingpedia API format
     */
    protected abstract resolveEntity(entityType : string,
                                    entityDisplay : string,
                                    hints : DisambiguationHints) : Promise<EntityRecord>;

    /* instanbul ignore next */
    /**
     * Resolve a `$context` variable to a concrete value.
     *
     * @param {string} variable - the variable name to lookup, including the `$context.` prefix
     * @returns {thingtalk.Ast.Value} - the resolved value
     */
    protected async resolveUserContext(variable : string) : Promise<Ast.Value> {
        throw new TypeError('Abstract method');
    }

    /* instanbul ignore next */
    /**
     * Compute the user's preferred unit to use when the program specifies an ambiguous unit
     * such as "degrees".
     *
     * @param {string} type - the type of unit to retrieve (e.g. "temperature"), or undefined
     *   if the user has no preference
     * @returns {string} - the preferred unit
     */
    getPreferredUnit(type : string) : string|undefined {
        throw new TypeError('Abstract method');
    }
}

stanford-oval / genie-toolkit / 2508

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous