1647

pending completion

Build # 1647

Build Type

Pull #464

travis-ci-com

Committed by

web-flow

Commit Message

Merge d6a136103 into 7976680a9

Pull Request Pull Request #464: evaluation: tokenize unresolved location names when comparing slots

Run Details

10819 of 23272 branches covered (46.49%)

Branch coverage included in aggregate %.

0 of 1 new or added line in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

25401 of 43954 relevant lines covered (57.79%)

529821.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

4.7

/lib/dataset-tools/evaluation/dialogue_evaluator.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>


import Stream from 'stream';
import deepEqual from 'deep-equal';
import * as Tp from 'thingpedia';
import * as ThingTalk from 'thingtalk';
import { Ast, } from 'thingtalk';
import assert from 'assert';

import * as Utils from '../../utils/misc-utils';
import { EntityMap } from '../../utils/entity-utils';
import * as I18n from '../../i18n';
import { EntityRecord, getBestEntityMatch } from '../../dialogue-agent/entity-linking/entity-finder';
import { stripOutTypeAnnotations } from './eval_utils';
import { ParserClient, PredictionResult } from '../../prediction/parserclient';
import { ParsedDialogue, DialogueTurn } from '../parsers';
import * as ThingTalkUtils from '../../utils/thingtalk';

// FIXME move this to a shared place with the simulation code
interface SimulationDatabase {
    has(key : string) : boolean;
    get(key : string) : Array<{ [key : string] : unknown }>|undefined;
}

interface DialogueEvaluatorOptions {
    thingpediaClient : Tp.BaseClient|null;
    locale : string;
    targetLanguage : string;
    tokenized : boolean;
    database ?: SimulationDatabase;

    debug ?: boolean;
}

export interface ExampleEvaluationResult {
    turns : number;
    ok : number;
    ok_slot : number;
    ok_initial : number;
    ok_initial_slot : number;
    ok_partial : number;
    ok_partial_slot : number;
    ok_prefix : number;
    ok_prefix_slot : number;
    ok_progress : number;
    ok_progress_slot : number;
}
export type EvaluationResult = ExampleEvaluationResult & {
    total : number,
    turns : number,
    [key : string] : number
};

const MINIBATCH_SIZE = 100;

class DialogueEvaluatorStream extends Stream.Transform {
    private _parser : ParserClient;
    private _tpClient : Tp.BaseClient|null;
    private _tokenizer : I18n.BaseTokenizer;
    private _options : DialogueEvaluatorOptions;
    private _locale : string;
    private _debug : boolean;
    private _tokenized : boolean;
    private _database : SimulationDatabase|undefined;
    private _cachedEntityMatches : Map<string, EntityRecord>;
    private _minibatch : Array<Promise<ExampleEvaluationResult>>;

    constructor(parser : ParserClient,
                options : DialogueEvaluatorOptions) {
        super({ objectMode: true });

        this._parser = parser;
        this._tpClient = options.thingpediaClient;
        this._tokenizer = I18n.get(options.locale).getTokenizer();

        this._options = options;
        this._locale = options.locale;
        this._debug = !!options.debug;
        this._tokenized = options.tokenized;
        this._database = options.database;

        this._cachedEntityMatches = new Map;

        this._minibatch = [];
    }

    private async _preprocess(sentence : string, contextEntities : EntityMap) {
        let tokenized;
        if (this._tokenized) {
            const tokens = sentence.split(' ');
            const entities = Utils.makeDummyEntities(sentence);
            tokenized = { tokens, entities };
        } else {
            tokenized = this._tokenizer.tokenize(sentence);
        }
        Utils.renumberEntities(tokenized, contextEntities);
        return tokenized;
    }

    private _isWellKnownEntity(entityType : string) {
        switch (entityType) {
        case 'tt:username':
        case 'tt:hashtag':
        case 'tt:picture':
        case 'tt:url':
        case 'tt:email_address':
        case 'tt:phone_number':
        case 'tt:path_name':
        case 'tt:device':
        case 'tt:function':
            return true;
        default:
            return false;
        }
    }

    private _tokenizeSlot(value : string) {
        return this._tokenizer.tokenize(value).rawTokens.join(' ');
    }

    private async _resolveEntity(value : Ast.EntityValue) : Promise<EntityRecord> {
        if (this._isWellKnownEntity(value.type)) {
            assert(value.value);
            return { value: value.value!, name: value.display||'', canonical: value.value! };
        }

        const searchKey = value.display||value.value;
        assert(searchKey);
        const cacheKey = value.type + '/' + value.value + '/' + searchKey;
        let resolved = this._cachedEntityMatches.get(cacheKey);
        if (resolved)
            return resolved;

        if (this._database && this._database.has(value.type)) {
            // resolve as ID entity from the database (simulate issuing a query for it)
            const ids = this._database.get(value.type)!.map((entry) => {
                const id = entry.id as { value : string, display : string };
                return {
                    value: id.value,
                    name: id.display,
                    canonical: id.display.toLowerCase()
                };
            });
            if (value.value) {
                for (const id of ids) {
                    if (id.value === value.value) {
                        resolved = id;
                        break;
                    }
                }
            }
            if (!resolved)
                resolved = getBestEntityMatch(searchKey, value.type, ids);
            this._cachedEntityMatches.set(cacheKey, resolved);
            return resolved;
        }

        // resolve as regular Thingpedia entity
        const candidates = await this._tpClient!.lookupEntity(value.type, searchKey);
        resolved = getBestEntityMatch(searchKey, value.type, candidates.data);
        this._cachedEntityMatches.set(cacheKey, resolved);
        return resolved;
    }

    private async _valueToSlot(value : Ast.Value) : Promise<string> {
        // HACK
        if (value instanceof Ast.ComputationValue)
            return this._valueToSlot(value.operands[0]);
        if (value instanceof Ast.EntityValue) {
            const resolved = await this._resolveEntity(value);
            if (resolved)
                return resolved.canonical;
            return this._tokenizeSlot(value.display||'');
        }
        if (value instanceof Ast.BooleanValue)
            return value.value ? 'yes' : 'no';
        if (value instanceof Ast.LocationValue) {
            if (value.value instanceof Ast.RelativeLocation)
                return value.value.relativeTag;
            if (value.value instanceof Ast.AbsoluteLocation)
                return this._tokenizeSlot(value.value.display||'');
            // unresolved
            assert(value.value instanceof Ast.UnresolvedLocation);
            return this._tokenizeSlot(value.value.name);
        }
        if (value instanceof Ast.ContextRefValue)
            return 'context-' + value.name;

        if (value instanceof Ast.StringValue) {
            // "tokenize" the value, because the prediction will also be tokenized
            return this._tokenizeSlot(value.toJS());
        }

        // everything else (time, currency, number, enum), use JS value
        return String(value.toJS()).toLowerCase();
    }

    private async _extractSlots(state : Ast.Node) {
        const slotValues : Record<string, Ast.Value> = {};
        let currentDomain : string|undefined;

        function nameToSlot(domain : string, name : string) {
            if (name === 'id' || name === domain)
                return domain + '-name';
            const slotKey = domain + '-' + name.replace(/_/g, '-');
            return slotKey;
        }

        // note: this function relies on the precise visit order, in which an invocation
        // is visited before the boolean expressions that use the output of that invocation
        state.visit(new class extends ThingTalk.Ast.NodeVisitor {
            visitInvocation(invocation : Ast.Invocation) {
                const selector = invocation.selector;
                assert(selector instanceof Ast.DeviceSelector);
                const device = selector.kind;
                const domain = device.substring(device.lastIndexOf('.')+1).toLowerCase();
                currentDomain = domain;

                // delete all slots for this domain (they'll be set again right after)
                for (const arg of invocation.schema!.iterateArguments()) {
                    if (arg.name === currentDomain) {
                        // do not erase the "id" slot just because we have an action!
                        assert(arg.type.isEntity);
                        continue;
                    }
                    const slotKey = nameToSlot(domain, arg.name);
                    delete slotValues[slotKey];
                }

                for (const in_param of invocation.in_params) {
                    if (in_param.value.isUndefined || in_param.value.isVarRef)
                        continue;
                    const slotKey = nameToSlot(domain, in_param.name);
                    slotValues[slotKey] = in_param.value;
                }

                // do not recurse
                return false;
            }

            visitDialogueHistoryItem(item : Ast.DialogueHistoryItem) {
                // recurse only if this item comes from the user and not the agent
                return item.confirm !== 'proposed';
            }

            visitDontCareBooleanExpression(expr : Ast.DontCareBooleanExpression) {
                const slotKey = nameToSlot(currentDomain!, expr.name);
                slotValues[slotKey] = new ThingTalk.Ast.Value.Enum('dontcare');
                return false;
            }

            visitAtomBooleanExpression(expr : Ast.AtomBooleanExpression) {
                if (expr.value.isUndefined || expr.value.isVarRef)
                    return false;

                const slotKey = nameToSlot(currentDomain!, expr.name);
                if (expr.operator === 'in_array') // multiple values, pick the first one
                    slotValues[slotKey] = (expr.value as Ast.ArrayValue).value[0];
                else
                    slotValues[slotKey] = expr.value;
                return false;
            }

            visitNotBooleanExpression(expr : Ast.NotBooleanExpression) {
                // explicitly do not recurse into "not" operators
                return false;
            }

            visitOrBooleanExpression(expr : Ast.OrBooleanExpression) {
                // explicitly do not recurse into "or" operators
                // (unless they are an "or" of one operand)
                return expr.operands.length === 1;
            }
        });

        // remove train-name which is not in multiwoz
        delete slotValues['train-name'];

        // resolve entities and map Ast.Value to a string we can compare for equality
        const slotStrings : Record<string, string> = {};
        for (const key in slotValues)
            slotStrings[key] = await this._valueToSlot(slotValues[key]);

        return slotStrings;
    }

    private async _checkTurn(id : string, turn : DialogueTurn, turnIndex : number) : Promise<'ok' | 'ok_slot' | 'ok_syntax' | 'wrong_syntax'> {
        let context, contextCode, contextEntities;
        if (turnIndex > 0) {
            if (turn.intermediate_context) {
                context = await ThingTalkUtils.parse(turn.intermediate_context, this._options);
                assert(context instanceof Ast.DialogueState);
            } else {
                context = await ThingTalkUtils.parse(turn.context!, this._options);
                assert(context instanceof Ast.DialogueState);
                // apply the agent prediction to the context to get the state of the dialogue before
                // the user speaks
                const agentPrediction = await ThingTalkUtils.parse(turn.agent_target!, this._options);
                assert(agentPrediction instanceof Ast.DialogueState);
                context = ThingTalkUtils.computeNewState(context, agentPrediction, 'agent');
            }

            const userContext = ThingTalkUtils.prepareContextForPrediction(context, 'user');
            [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(userContext);
        } else {
            context = null;
            contextCode = ['null'];
            contextEntities = {};
        }

        const { tokens, entities } = await this._preprocess(turn.user, contextEntities);
        const goldUserTarget = await ThingTalkUtils.parse(turn.user_target, this._options);
        assert(goldUserTarget instanceof Ast.DialogueState);
        const goldUserState = ThingTalkUtils.computeNewState(context, goldUserTarget, 'user');
        const goldSlots = await this._extractSlots(goldUserState);

        const targetCode = ThingTalkUtils.serializePrediction(goldUserTarget, tokens, entities, {
           locale: this._locale,
        }).join(' ');

        const parsed : PredictionResult = await this._parser.sendUtterance(tokens.join(' '), contextCode, contextEntities, {
            tokenized: true,
            skip_typechecking: true,
            example_id: id + '/' + turnIndex,
        });

        const predictions = parsed.candidates
            .filter((beam) => beam.score !== 'Infinity') // ignore exact matches
            .map((beam) => beam.code);

        if (predictions.length === 0) {
            if (this._debug)
                console.log(`${id}:${turnIndex}\twrong_syntax\t${contextCode.join(' ')}\t${turn.user}\tfailed\t${targetCode}`);
            return 'wrong_syntax';
        }

        const choice : string[] = predictions[0];

        // first check if the program parses and typechecks (no hope otherwise)
        let predictedUserTarget : Ast.Input;
        try {
            predictedUserTarget = await ThingTalkUtils.parsePrediction(choice, entities, this._options, true);
            assert(predictedUserTarget instanceof Ast.DialogueState);
        } catch(e) {
            if (this._debug)
                console.log(`${id}:${turnIndex}\twrong_syntax\t${contextCode.join(' ')}\t${turn.user}\t${choice.join(' ')}\t${targetCode}`);
            return 'wrong_syntax';
        }

        const predictedUserState = ThingTalkUtils.computeNewState(context, predictedUserTarget, 'user');
        let predictedSlots;
        try {
            predictedSlots = await this._extractSlots(predictedUserState);
        } catch(e) {
            console.error(predictedUserTarget.prettyprint());
            throw e;
        }

        // do some light syntactic normalization
        const choiceString : string = Array.from(stripOutTypeAnnotations(choice)).join(' ');

        // do the actual normalization, using the full ThingTalk algorithm
        // we pass "ignoreSentence: true", which means strings are tokenized and then put in the
        // program regardless of what the sentence contains (because the neural network might
        // get creative in copying, and we don't want to crash here)
        const normalized = ThingTalkUtils.serializePrediction(predictedUserTarget, tokens, entities, {
           locale: this._locale,
           ignoreSentence: true
        }).join(' ');

        // check that by normalizing we did not accidentally mark wrong a program that
        // was correct before
        if (choiceString === targetCode && normalized !== targetCode) {
            console.error();
            console.error('NORMALIZATION ERROR');
            console.error(targetCode);
            console.error(normalized);
            console.error(choice.join(' '));
            console.error(choiceString);
            throw new Error('Normalization Error');
        }

        if (normalized === targetCode) {
            if (this._debug)
                console.log(`${id}:${turnIndex}\tok\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);
            if (!deepEqual(goldSlots, predictedSlots, { strict: true })) {
                console.error(goldSlots, predictedSlots);
                throw new Error(`Program matches but slots do not`);
            }
            return 'ok';
        } else if (deepEqual(goldSlots, predictedSlots, { strict: true })) {
            if (this._debug)
                console.log(`${id}:${turnIndex}\tok_slot\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);
            return 'ok_slot';
        } else {
            if (this._debug)
                console.log(`${id}:${turnIndex}\tok_syntax\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);
            return 'ok_syntax';
        }
    }

    private async _evaluate(dialogue : ParsedDialogue) {
        let prefix_full = 0;
        let correct_full = 0;
        let prefix_slot = 0;
        let correct_slot = 0;
        let failed_full = false, failed_slot = false;
        for (let i = 0; i < dialogue.length; i++) {
            const turn = dialogue[i];
            let ok;
            try {
                ok = await this._checkTurn(dialogue.id, turn, i);
            } catch(e) {
                console.error(dialogue.id, turn);
                throw e;
            }
            if (ok === 'ok') {
                correct_full += 1;
                correct_slot += 1;
                if (!failed_slot)
                    prefix_slot += 1;
                if (!failed_full)
                    prefix_full += 1;
            } else if (ok === 'ok_slot') {
                correct_slot += 1;
                if (!failed_slot)
                    prefix_slot += 1;
                failed_full = true;
            } else {
                failed_full = true;
                failed_slot = true;
            }
        }

        const ret : ExampleEvaluationResult = {
            turns: dialogue.length,
            ok: Number(correct_full === dialogue.length),
            ok_slot: Number(correct_slot === dialogue.length),
            ok_initial: Number(prefix_full >= 1),
            ok_initial_slot: Number(prefix_slot >= 1),
            ok_partial: Number(correct_full),
            ok_partial_slot: Number(correct_slot),
            ok_prefix: Number(prefix_full),
            ok_prefix_slot: Number(prefix_slot),
            ok_progress: Number(prefix_full),
            ok_progress_slot: Number(prefix_slot),
        };

        if (this._debug) {
            let message = String(dialogue.id);
            for (const key in ret) {
                if (key === 'turns')
                    continue;
                message += '\t' + ret[key as keyof ExampleEvaluationResult];
            }
            console.log(message);
        }

        return ret;
    }

    private async _flushMinibatch() {
        for (const res of await Promise.all(this._minibatch))
            this.push(res);
        this._minibatch = [];
    }

    private async _pushDialogue(dialog : ParsedDialogue) {
        this._minibatch.push(this._evaluate(dialog));
        if (this._minibatch.length >= MINIBATCH_SIZE)
            await this._flushMinibatch();
    }

    _transform(dialog : ParsedDialogue, encoding : BufferEncoding, callback : (err : Error|null) => void) {
        this._pushDialogue(dialog).then(() => callback(null), (err) => callback(err));
    }

    _flush(callback : (err : Error|null) => void) {
        this._flushMinibatch().then(() => callback(null), (err) => callback(err));
    }
}

const KEYS : Array<keyof ExampleEvaluationResult> = [
    'ok', 'ok_slot', 'ok_initial', 'ok_initial_slot',
    'ok_partial', 'ok_partial_slot', 'ok_prefix', 'ok_prefix_slot',
    'ok_progress', 'ok_progress_slot'
];
const BY_TURN_KEYS : Array<keyof ExampleEvaluationResult> = ['ok_partial', 'ok_partial_slot', 'ok_prefix', 'ok_prefix_slot'];
const BY_DIALOGUE_KEYS : Array<keyof ExampleEvaluationResult> = ['ok', 'ok_slot', 'ok_initial', 'ok_initial_slot', 'ok_progress', 'ok_progress_slot'];
class CollectDialogueStatistics extends Stream.Writable {
    private _buffer : EvaluationResult;

    constructor() {
        super({ objectMode: true });

        this._buffer = {
            total: 0,
            turns: 0,
        } as EvaluationResult;
        for (const key of KEYS)
            this._buffer[key] = 0;
    }

    _write(sample : EvaluationResult, encoding : BufferEncoding, callback : () => void) {
        this._buffer.total ++;
        this._buffer.turns += sample.turns;
        for (const key of KEYS)
            this._buffer[key] += sample[key];
        callback();
    }

    _final(callback : () => void) {
        // convert to percentages
        for (const key of BY_DIALOGUE_KEYS)
            this._buffer[key] /= this._buffer.total;
        for (const key of BY_TURN_KEYS)
            this._buffer[key] /= this._buffer.turns;
        callback();
    }

    read() {
        return new Promise<EvaluationResult>((resolve, reject) => {
            this.on('finish', () => resolve(this._buffer));
            this.on('error', reject);
        });
    }
}

export {
    KEYS,
    DialogueEvaluatorStream,
    CollectDialogueStatistics
};

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21
22	import Stream from 'stream';	60✔
23	import deepEqual from 'deep-equal';	60✔
24	import * as Tp from 'thingpedia';
25	import * as ThingTalk from 'thingtalk';	60✔
26	import { Ast, } from 'thingtalk';	60✔
27	import assert from 'assert';	60✔
28
29	import * as Utils from '../../utils/misc-utils';	60✔
30	import { EntityMap } from '../../utils/entity-utils';
31	import * as I18n from '../../i18n';	60✔
32	import { EntityRecord, getBestEntityMatch } from '../../dialogue-agent/entity-linking/entity-finder';	60✔
33	import { stripOutTypeAnnotations } from './eval_utils';	60✔
34	import { ParserClient, PredictionResult } from '../../prediction/parserclient';
35	import { ParsedDialogue, DialogueTurn } from '../parsers';
36	import * as ThingTalkUtils from '../../utils/thingtalk';	60✔
37
38	// FIXME move this to a shared place with the simulation code
39	interface SimulationDatabase {
40	has(key : string) : boolean;
41	get(key : string) : Array<{ [key : string] : unknown }>\|undefined;
42	}
43
44	interface DialogueEvaluatorOptions {
45	thingpediaClient : Tp.BaseClient\|null;
46	locale : string;
47	targetLanguage : string;
48	tokenized : boolean;
49	database ?: SimulationDatabase;
50
51	debug ?: boolean;
52	}
53
54	export interface ExampleEvaluationResult {
55	turns : number;
56	ok : number;
57	ok_slot : number;
58	ok_initial : number;
59	ok_initial_slot : number;
60	ok_partial : number;
61	ok_partial_slot : number;
62	ok_prefix : number;
63	ok_prefix_slot : number;
64	ok_progress : number;
65	ok_progress_slot : number;
66	}
67	export type EvaluationResult = ExampleEvaluationResult & {
68	total : number,
69	turns : number,
70	[key : string] : number
71	};
72
73	const MINIBATCH_SIZE = 100;	60✔
74
75	class DialogueEvaluatorStream extends Stream.Transform {
76	private _parser : ParserClient;
77	private _tpClient : Tp.BaseClient\|null;
78	private _tokenizer : I18n.BaseTokenizer;
79	private _options : DialogueEvaluatorOptions;
80	private _locale : string;
81	private _debug : boolean;
82	private _tokenized : boolean;
83	private _database : SimulationDatabase\|undefined;
84	private _cachedEntityMatches : Map<string, EntityRecord>;
85	private _minibatch : Array<Promise<ExampleEvaluationResult>>;
86
87	constructor(parser : ParserClient,
88	options : DialogueEvaluatorOptions) {
89	super({ objectMode: true });	×
90
91	this._parser = parser;	×
92	this._tpClient = options.thingpediaClient;	×
93	this._tokenizer = I18n.get(options.locale).getTokenizer();	×
94
95	this._options = options;	×
96	this._locale = options.locale;	×
97	this._debug = !!options.debug;	×
98	this._tokenized = options.tokenized;	×
99	this._database = options.database;	×
100
101	this._cachedEntityMatches = new Map;	×
102
103	this._minibatch = [];	×
104	}
105
106	private async _preprocess(sentence : string, contextEntities : EntityMap) {
107	let tokenized;
108	if (this._tokenized) {	×
109	const tokens = sentence.split(' ');	×
110	const entities = Utils.makeDummyEntities(sentence);	×
111	tokenized = { tokens, entities };	×
112	} else {
113	tokenized = this._tokenizer.tokenize(sentence);	×
114	}
115	Utils.renumberEntities(tokenized, contextEntities);	×
116	return tokenized;	×
117	}
118
119	private _isWellKnownEntity(entityType : string) {
120	switch (entityType) {	×
121	case 'tt:username':	×
122	case 'tt:hashtag':
123	case 'tt:picture':
124	case 'tt:url':
125	case 'tt:email_address':
126	case 'tt:phone_number':
127	case 'tt:path_name':
128	case 'tt:device':
129	case 'tt:function':
130	return true;	×
131	default:
132	return false;	×
133	}
134	}
135
136	private _tokenizeSlot(value : string) {
137	return this._tokenizer.tokenize(value).rawTokens.join(' ');	×
138	}
139
140	private async _resolveEntity(value : Ast.EntityValue) : Promise<EntityRecord> {
141	if (this._isWellKnownEntity(value.type)) {	×
142	assert(value.value);	×
143	return { value: value.value!, name: value.display\|\|'', canonical: value.value! };	×
144	}
145
146	const searchKey = value.display\|\|value.value;	×
147	assert(searchKey);	×
148	const cacheKey = value.type + '/' + value.value + '/' + searchKey;	×
149	let resolved = this._cachedEntityMatches.get(cacheKey);	×
150	if (resolved)	×
151	return resolved;	×
152
153	if (this._database && this._database.has(value.type)) {	×
154	// resolve as ID entity from the database (simulate issuing a query for it)
155	const ids = this._database.get(value.type)!.map((entry) => {	×
156	const id = entry.id as { value : string, display : string };	×
157	return {	×
158	value: id.value,
159	name: id.display,
160	canonical: id.display.toLowerCase()
161	};
162	});
163	if (value.value) {	×
164	for (const id of ids) {	×
165	if (id.value === value.value) {	×
166	resolved = id;	×
167	break;	×
168	}
169	}
170	}
171	if (!resolved)	×
172	resolved = getBestEntityMatch(searchKey, value.type, ids);	×
173	this._cachedEntityMatches.set(cacheKey, resolved);	×
174	return resolved;	×
175	}
176
177	// resolve as regular Thingpedia entity
178	const candidates = await this._tpClient!.lookupEntity(value.type, searchKey);	×
179	resolved = getBestEntityMatch(searchKey, value.type, candidates.data);	×
180	this._cachedEntityMatches.set(cacheKey, resolved);	×
181	return resolved;	×
182	}
183
184	private async _valueToSlot(value : Ast.Value) : Promise<string> {
185	// HACK
186	if (value instanceof Ast.ComputationValue)	×
187	return this._valueToSlot(value.operands[0]);	×
188	if (value instanceof Ast.EntityValue) {	×
189	const resolved = await this._resolveEntity(value);	×
190	if (resolved)	×
191	return resolved.canonical;	×
192	return this._tokenizeSlot(value.display\|\|'');	×
193	}
194	if (value instanceof Ast.BooleanValue)	×
195	return value.value ? 'yes' : 'no';	×
196	if (value instanceof Ast.LocationValue) {	×
197	if (value.value instanceof Ast.RelativeLocation)	×
198	return value.value.relativeTag;	×
199	if (value.value instanceof Ast.AbsoluteLocation)	×
200	return this._tokenizeSlot(value.value.display\|\|'');	×
201	// unresolved
202	assert(value.value instanceof Ast.UnresolvedLocation);	×
NEW 203	return this._tokenizeSlot(value.value.name);	×
204	}
205	if (value instanceof Ast.ContextRefValue)	×
206	return 'context-' + value.name;	×
207
208	if (value instanceof Ast.StringValue) {	×
209	// "tokenize" the value, because the prediction will also be tokenized
210	return this._tokenizeSlot(value.toJS());	×
211	}
212
213	// everything else (time, currency, number, enum), use JS value
214	return String(value.toJS()).toLowerCase();	×
215	}
216
217	private async _extractSlots(state : Ast.Node) {
218	const slotValues : Record<string, Ast.Value> = {};	×
219	let currentDomain : string\|undefined;
220
221	function nameToSlot(domain : string, name : string) {
222	if (name === 'id' \|\| name === domain)	×
223	return domain + '-name';	×
224	const slotKey = domain + '-' + name.replace(/_/g, '-');	×
225	return slotKey;	×
226	}
227
228	// note: this function relies on the precise visit order, in which an invocation
229	// is visited before the boolean expressions that use the output of that invocation
230	state.visit(new class extends ThingTalk.Ast.NodeVisitor {	×
231	visitInvocation(invocation : Ast.Invocation) {
232	const selector = invocation.selector;	×
233	assert(selector instanceof Ast.DeviceSelector);	×
234	const device = selector.kind;	×
235	const domain = device.substring(device.lastIndexOf('.')+1).toLowerCase();	×
236	currentDomain = domain;	×
237
238	// delete all slots for this domain (they'll be set again right after)
239	for (const arg of invocation.schema!.iterateArguments()) {	×
240	if (arg.name === currentDomain) {	×
241	// do not erase the "id" slot just because we have an action!
242	assert(arg.type.isEntity);	×
243	continue;	×
244	}
245	const slotKey = nameToSlot(domain, arg.name);	×
246	delete slotValues[slotKey];	×
247	}
248
249	for (const in_param of invocation.in_params) {	×
250	if (in_param.value.isUndefined \|\| in_param.value.isVarRef)	×
251	continue;	×
252	const slotKey = nameToSlot(domain, in_param.name);	×
253	slotValues[slotKey] = in_param.value;	×
254	}
255
256	// do not recurse
257	return false;	×
258	}
259
260	visitDialogueHistoryItem(item : Ast.DialogueHistoryItem) {
261	// recurse only if this item comes from the user and not the agent
262	return item.confirm !== 'proposed';	×
263	}
264
265	visitDontCareBooleanExpression(expr : Ast.DontCareBooleanExpression) {
266	const slotKey = nameToSlot(currentDomain!, expr.name);	×
267	slotValues[slotKey] = new ThingTalk.Ast.Value.Enum('dontcare');	×
268	return false;	×
269	}
270
271	visitAtomBooleanExpression(expr : Ast.AtomBooleanExpression) {
272	if (expr.value.isUndefined \|\| expr.value.isVarRef)	×
273	return false;	×
274
275	const slotKey = nameToSlot(currentDomain!, expr.name);	×
276	if (expr.operator === 'in_array') // multiple values, pick the first one	×
277	slotValues[slotKey] = (expr.value as Ast.ArrayValue).value[0];	×
278	else
279	slotValues[slotKey] = expr.value;	×
280	return false;	×
281	}
282
283	visitNotBooleanExpression(expr : Ast.NotBooleanExpression) {
284	// explicitly do not recurse into "not" operators
285	return false;	×
286	}
287
288	visitOrBooleanExpression(expr : Ast.OrBooleanExpression) {
289	// explicitly do not recurse into "or" operators
290	// (unless they are an "or" of one operand)
291	return expr.operands.length === 1;	×
292	}
293	});
294
295	// remove train-name which is not in multiwoz
296	delete slotValues['train-name'];	×
297
298	// resolve entities and map Ast.Value to a string we can compare for equality
299	const slotStrings : Record<string, string> = {};	×
300	for (const key in slotValues)	×
301	slotStrings[key] = await this._valueToSlot(slotValues[key]);	×
302
303	return slotStrings;	×
304	}
305
306	private async _checkTurn(id : string, turn : DialogueTurn, turnIndex : number) : Promise<'ok' \| 'ok_slot' \| 'ok_syntax' \| 'wrong_syntax'> {
307	let context, contextCode, contextEntities;
308	if (turnIndex > 0) {	×
309	if (turn.intermediate_context) {	×
310	context = await ThingTalkUtils.parse(turn.intermediate_context, this._options);	×
311	assert(context instanceof Ast.DialogueState);	×
312	} else {
313	context = await ThingTalkUtils.parse(turn.context!, this._options);	×
314	assert(context instanceof Ast.DialogueState);	×
315	// apply the agent prediction to the context to get the state of the dialogue before
316	// the user speaks
317	const agentPrediction = await ThingTalkUtils.parse(turn.agent_target!, this._options);	×
318	assert(agentPrediction instanceof Ast.DialogueState);	×
319	context = ThingTalkUtils.computeNewState(context, agentPrediction, 'agent');	×
320	}
321
322	const userContext = ThingTalkUtils.prepareContextForPrediction(context, 'user');	×
323	[contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(userContext);	×
324	} else {
325	context = null;	×
326	contextCode = ['null'];	×
327	contextEntities = {};	×
328	}
329
330	const { tokens, entities } = await this._preprocess(turn.user, contextEntities);	×
331	const goldUserTarget = await ThingTalkUtils.parse(turn.user_target, this._options);	×
332	assert(goldUserTarget instanceof Ast.DialogueState);	×
333	const goldUserState = ThingTalkUtils.computeNewState(context, goldUserTarget, 'user');	×
334	const goldSlots = await this._extractSlots(goldUserState);	×
335
336	const targetCode = ThingTalkUtils.serializePrediction(goldUserTarget, tokens, entities, {	×
337	locale: this._locale,
338	}).join(' ');
339
340	const parsed : PredictionResult = await this._parser.sendUtterance(tokens.join(' '), contextCode, contextEntities, {	×
341	tokenized: true,
342	skip_typechecking: true,
343	example_id: id + '/' + turnIndex,
344	});
345
346	const predictions = parsed.candidates	×
347	.filter((beam) => beam.score !== 'Infinity') // ignore exact matches	×
348	.map((beam) => beam.code);	×
349
350	if (predictions.length === 0) {	×
351	if (this._debug)	×
352	console.log(`${id}:${turnIndex}\twrong_syntax\t${contextCode.join(' ')}\t${turn.user}\tfailed\t${targetCode}`);	×
353	return 'wrong_syntax';	×
354	}
355
356	const choice : string[] = predictions[0];	×
357
358	// first check if the program parses and typechecks (no hope otherwise)
359	let predictedUserTarget : Ast.Input;
360	try {	×
361	predictedUserTarget = await ThingTalkUtils.parsePrediction(choice, entities, this._options, true);	×
362	assert(predictedUserTarget instanceof Ast.DialogueState);	×
363	} catch(e) {
364	if (this._debug)	×
365	console.log(`${id}:${turnIndex}\twrong_syntax\t${contextCode.join(' ')}\t${turn.user}\t${choice.join(' ')}\t${targetCode}`);	×
366	return 'wrong_syntax';	×
367	}
368
369	const predictedUserState = ThingTalkUtils.computeNewState(context, predictedUserTarget, 'user');	×
370	let predictedSlots;
371	try {	×
372	predictedSlots = await this._extractSlots(predictedUserState);	×
373	} catch(e) {
374	console.error(predictedUserTarget.prettyprint());	×
375	throw e;	×
376	}
377
378	// do some light syntactic normalization
379	const choiceString : string = Array.from(stripOutTypeAnnotations(choice)).join(' ');	×
380
381	// do the actual normalization, using the full ThingTalk algorithm
382	// we pass "ignoreSentence: true", which means strings are tokenized and then put in the
383	// program regardless of what the sentence contains (because the neural network might
384	// get creative in copying, and we don't want to crash here)
385	const normalized = ThingTalkUtils.serializePrediction(predictedUserTarget, tokens, entities, {	×
386	locale: this._locale,
387	ignoreSentence: true
388	}).join(' ');
389
390	// check that by normalizing we did not accidentally mark wrong a program that
391	// was correct before
392	if (choiceString === targetCode && normalized !== targetCode) {	×
393	console.error();	×
394	console.error('NORMALIZATION ERROR');	×
395	console.error(targetCode);	×
396	console.error(normalized);	×
397	console.error(choice.join(' '));	×
398	console.error(choiceString);	×
399	throw new Error('Normalization Error');	×
400	}
401
402	if (normalized === targetCode) {	×
403	if (this._debug)	×
404	console.log(`${id}:${turnIndex}\tok\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);	×
405	if (!deepEqual(goldSlots, predictedSlots, { strict: true })) {	×
406	console.error(goldSlots, predictedSlots);	×
407	throw new Error(`Program matches but slots do not`);	×
408	}
409	return 'ok';	×
410	} else if (deepEqual(goldSlots, predictedSlots, { strict: true })) {	×
411	if (this._debug)	×
412	console.log(`${id}:${turnIndex}\tok_slot\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);	×
413	return 'ok_slot';	×
414	} else {
415	if (this._debug)	×
416	console.log(`${id}:${turnIndex}\tok_syntax\t${contextCode.join(' ')}\t${turn.user}\t${normalized}\t${targetCode}`);	×
417	return 'ok_syntax';	×
418	}
419	}
420
421	private async _evaluate(dialogue : ParsedDialogue) {
422	let prefix_full = 0;	×
423	let correct_full = 0;	×
424	let prefix_slot = 0;	×
425	let correct_slot = 0;	×
426	let failed_full = false, failed_slot = false;	×
427	for (let i = 0; i < dialogue.length; i++) {	×
428	const turn = dialogue[i];	×
429	let ok;
430	try {	×
431	ok = await this._checkTurn(dialogue.id, turn, i);	×
432	} catch(e) {
433	console.error(dialogue.id, turn);	×
434	throw e;	×
435	}
436	if (ok === 'ok') {	×
437	correct_full += 1;	×
438	correct_slot += 1;	×
439	if (!failed_slot)	×
440	prefix_slot += 1;	×
441	if (!failed_full)	×
442	prefix_full += 1;	×
443	} else if (ok === 'ok_slot') {	×
444	correct_slot += 1;	×
445	if (!failed_slot)	×
446	prefix_slot += 1;	×
447	failed_full = true;	×
448	} else {
449	failed_full = true;	×
450	failed_slot = true;	×
451	}
452	}
453
454	const ret : ExampleEvaluationResult = {	×
455	turns: dialogue.length,
456	ok: Number(correct_full === dialogue.length),
457	ok_slot: Number(correct_slot === dialogue.length),
458	ok_initial: Number(prefix_full >= 1),
459	ok_initial_slot: Number(prefix_slot >= 1),
460	ok_partial: Number(correct_full),
461	ok_partial_slot: Number(correct_slot),
462	ok_prefix: Number(prefix_full),
463	ok_prefix_slot: Number(prefix_slot),
464	ok_progress: Number(prefix_full),
465	ok_progress_slot: Number(prefix_slot),
466	};
467
468	if (this._debug) {	×
469	let message = String(dialogue.id);	×
470	for (const key in ret) {	×
471	if (key === 'turns')	×
472	continue;	×
473	message += '\t' + ret[key as keyof ExampleEvaluationResult];	×
474	}
475	console.log(message);	×
476	}
477
478	return ret;	×
479	}
480
481	private async _flushMinibatch() {
482	for (const res of await Promise.all(this._minibatch))	×
483	this.push(res);	×
484	this._minibatch = [];	×
485	}
486
487	private async _pushDialogue(dialog : ParsedDialogue) {
488	this._minibatch.push(this._evaluate(dialog));	×
489	if (this._minibatch.length >= MINIBATCH_SIZE)	×
490	await this._flushMinibatch();	×
491	}
492
493	_transform(dialog : ParsedDialogue, encoding : BufferEncoding, callback : (err : Error\|null) => void) {
494	this._pushDialogue(dialog).then(() => callback(null), (err) => callback(err));	×
495	}
496
497	_flush(callback : (err : Error\|null) => void) {
498	this._flushMinibatch().then(() => callback(null), (err) => callback(err));	×
499	}
500	}
501
502	const KEYS : Array<keyof ExampleEvaluationResult> = [	60✔
503	'ok', 'ok_slot', 'ok_initial', 'ok_initial_slot',
504	'ok_partial', 'ok_partial_slot', 'ok_prefix', 'ok_prefix_slot',
505	'ok_progress', 'ok_progress_slot'
506	];
507	const BY_TURN_KEYS : Array<keyof ExampleEvaluationResult> = ['ok_partial', 'ok_partial_slot', 'ok_prefix', 'ok_prefix_slot'];	60✔
508	const BY_DIALOGUE_KEYS : Array<keyof ExampleEvaluationResult> = ['ok', 'ok_slot', 'ok_initial', 'ok_initial_slot', 'ok_progress', 'ok_progress_slot'];	60✔
509	class CollectDialogueStatistics extends Stream.Writable {
510	private _buffer : EvaluationResult;
511
512	constructor() {
513	super({ objectMode: true });	×
514
515	this._buffer = {	×
516	total: 0,
517	turns: 0,
518	} as EvaluationResult;
519	for (const key of KEYS)	×
520	this._buffer[key] = 0;	×
521	}
522
523	_write(sample : EvaluationResult, encoding : BufferEncoding, callback : () => void) {
524	this._buffer.total ++;	×
525	this._buffer.turns += sample.turns;	×
526	for (const key of KEYS)	×
527	this._buffer[key] += sample[key];	×
528	callback();	×
529	}
530
531	_final(callback : () => void) {
532	// convert to percentages
533	for (const key of BY_DIALOGUE_KEYS)	×
534	this._buffer[key] /= this._buffer.total;	×
535	for (const key of BY_TURN_KEYS)	×
536	this._buffer[key] /= this._buffer.turns;	×
537	callback();	×
538	}
539
540	read() {
541	return new Promise<EvaluationResult>((resolve, reject) => {	×
542	this.on('finish', () => resolve(this._buffer));	×
543	this.on('error', reject);	×
544	});
545	}
546	}
547
548	export {
549	KEYS,	60✔
550	DialogueEvaluatorStream,	60✔
551	CollectDialogueStatistics	60✔
552	};

stanford-oval / genie-toolkit / 1647

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous