1232

pending completion

Build # 1232

Build Type

Pull #282

travis-ci-com

Committed by

web-flow

Commit Message

Merge ec44c6e51 into 0b2c72916

Pull Request Pull Request #282: All changes made for cikm camera ready

Run Details

6629 of 12702 branches covered (52.19%)

Branch coverage included in aggregate %.

257 of 300 new or added lines in 23 files covered. (85.67%)

285 existing lines in 16 files now uncovered.

15326 of 24419 relevant lines covered (62.76%)

2000753.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.57

/lib/dataset-tools/evaluation/sentence_evaluator.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const assert = require('assert');
const Stream = require('stream');
const ThingTalk = require('thingtalk');

const Utils = require('../../utils/misc-utils');
const { requoteProgram, getFunctions, getDevices } = require('../requoting');
const { stripOutTypeAnnotations, normalizeKeywordParams } = require('./eval_utils');
const I18n = require('../../i18n');

function iterEquals(iterable1, iterable2) {
    let iter1 = iterable1[Symbol.iterator]();
    let iter2 = iterable2[Symbol.iterator]();
    for (;;) {
        let { value: value1, done: done1 } = iter1.next();
        let { value: value2, done: done2 } = iter2.next();
        if (done1 !== done2)
            return false;
        if (done1)
            break;
        if (value1 !== value2)
            return false;
    }
    return true;
}

const COMPLEXITY_METRICS = {
    num_params(id, code) {
        let params = 0;
        let joins = 0;
        let inString = false;
        for (let token of code.split(' ')) {
            if (token === '"')
                inString = !inString;
            if (inString)
                continue;

            // distance is computed by geo, which will be counted
            if (token.startsWith('param:') && !token.startsWith('param:distance'))
                params ++;
            else if (token === 'join')
                joins ++;
        }

        return params + joins;
    },

    turn_number(id, code) {
        const match = /\/([0-9]+)(?:-[0-9]+)*$/.exec(id);
        return parseInt(match[1]);
    }
};


class SentenceEvaluator {
    constructor(locale, parser, schemaRetriever, tokenized, debug, complexityMetric, ex) {
        this._parser = parser;
        this._tokenized = tokenized;
        this._debug = debug;
        this._schemas = schemaRetriever;
        this._tokenizer = I18n.get(locale).getTokenizer();

        this._id = ex.id;
        this._context = ex.context;
        this._preprocessed = ex.preprocessed;
        this._targetPrograms = ex.target_code;
        this._predictions = ex.predictions;

        if (complexityMetric)
            this._computeComplexity = COMPLEXITY_METRICS[complexityMetric];
    }

    _hasNumeric(code) {
        for (let token of code.split(' ')) {
            if (!isNaN(token))
                return true;
            if (token === '>=' || token === '<=')
                return true;
            if (token.startsWith('NUMBER_'))
                return true;
        }
        return false;
    }

    async evaluate() {

        const result = {
            id: this._id,
            preprocessed: this._preprocessed,
            target_code: this._targetPrograms,
            target_devices: [],
            ok: [],
            ok_without_param: [],
            ok_function: [],
            ok_device: [],
            ok_num_function: [],
            ok_syntax: [],

            is_primitive: false,
            complexity: undefined,
            has_numeric: false
        };

        let contextCode = undefined, contextEntities = {};
        if (this._context !== undefined) {
            contextCode = this._context.split(' ');
            contextEntities = Utils.makeDummyEntities(this._context);
        }

        let entities;
        if (this._tokenized) {
            entities = Utils.makeDummyEntities(this._preprocessed);
            Object.assign(entities, contextEntities);
        } else {
            const tokenized = await this._tokenizer.tokenize(this._preprocessed);
            Utils.renumberEntities(tokenized, contextEntities);
            entities = tokenized.entities;
        }

        assert(Array.isArray(this._targetPrograms));
        assert(this._targetPrograms.length > 0);

        const untypedTargetCode = [];
        const normalEntities = {};
        let firstTargetCode = this._targetPrograms[0];
        try {
            const sequence = firstTargetCode.split(' ');
            const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);
            await parsed.typecheck(this._schemas);
            const normalized = ThingTalk.NNSyntax.toNN(parsed, this._preprocessed, normalEntities, {
                allocateEntities: true,
                typeAnnotations: false
            });
            untypedTargetCode.push(normalized.join(' '));
        } catch(e) {
            // if the target_code did not parse due to missing functions in thingpedia, ignore it
            if (e.message.indexOf('has no query') >= 0 || e.message.indexOf('has no action') >= 0)
                return null;

            console.error(this._id, this._preprocessed, this._targetPrograms);
            throw e;
        }
        if (this._computeComplexity)
            result.complexity = this._computeComplexity(this._id, this._targetPrograms[0]);
        else
            result.complexity = 0;
        result.has_numeric = this._hasNumeric(this._targetPrograms[0]);

        // check all other target codes (sanity check)
        for (let i = 1; i < this._targetPrograms.length; i++) {
            try {
                const sequence = this._targetPrograms[i].split(' ');
                const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);
                await parsed.typecheck(this._schemas);
                const normalized = ThingTalk.NNSyntax.toNN(parsed, this._preprocessed, normalEntities, {
                    allocateEntities: true,
                    typeAnnotations: false
                });
                untypedTargetCode.push(normalized.join(' '));
            } catch(e) {
                console.error(this._id, this._preprocessed, this._targetPrograms);
                throw e;
            }
        }

        const requotedGold = untypedTargetCode.map((code) => Array.from(requoteProgram(code)));
        const goldFunctions = untypedTargetCode.map((code) => Array.from(getFunctions(code)));
        const goldDevices = untypedTargetCode.map((code) => Array.from(getDevices(code)));
        result.is_primitive = goldFunctions[0].length === 1;
        result.target_devices = goldDevices[0];

        let first = true;
        let ok = false, ok_without_param = false, ok_function = false,
            ok_device = false, ok_num_function = false, ok_syntax = false;

        let predictions;
        if (this._predictions) {
            predictions = this._predictions;
        } else {
            try {
                const parsed = await this._parser.sendUtterance(this._preprocessed, contextCode, contextEntities, {
                    tokenized: this._tokenized,
                    skip_typechecking: true
                });
                if (!entities)
                    entities = parsed.entities;

                predictions = parsed.candidates
                    .filter((beam) => beam.score !== 'Infinity') // ignore exact matches
                    .map((beam) => beam.code);
            } catch(e) {
                console.error(`Sentence ${this._id} failed to predict`);
                console.error(e);
                predictions = [[]];
            }
        }

        for (let beam of predictions) {
            let target = Array.from(stripOutTypeAnnotations(this._targetPrograms[0].split(' '))).join(' ');
            let prediction = Array.from(stripOutTypeAnnotations(beam)).join(' ');

            // first check if the program parses and typechecks (no hope otherwise)
            try {
                const parsed = ThingTalk.NNSyntax.fromNN(beam, entities);
                await parsed.typecheck(this._schemas);
                beam = ThingTalk.NNSyntax.toNN(parsed, beam, normalEntities, { allocateEntities: true });
            } catch(e) {
                // push the previous result, so the stats
                // stay cumulative along the beam

                result.ok.push(ok);
                result.ok_without_param.push(ok_without_param);
                result.ok_function.push(ok_function);
                result.ok_device.push(ok_device);
                result.ok_num_function.push(ok_num_function);
                result.ok_syntax.push(ok_syntax);
                if (first && this._debug)
                    console.log(`${this._id}\twrong_syntax\t${this._preprocessed}\t${target}\t${prediction}`);
                first = false;
                continue;
            }
            ok_syntax = true;

            const normalized = normalizeKeywordParams(Array.from(stripOutTypeAnnotations(beam)));
            const code = normalized.join(' ');

            let beam_ok = false, beam_ok_without_param = false, beam_ok_function = false,
                beam_ok_device = false, beam_ok_num_function = false;
            let result_string = 'ok_syntax';

            for (let referenceId = 0; referenceId < this._targetPrograms.length; referenceId++) {
                if (code === untypedTargetCode[referenceId]) {
                    // we have a match!

                    beam_ok = true;
                    beam_ok_without_param = true;
                    beam_ok_function = true;
                    beam_ok_device = true;
                    beam_ok_num_function = true;
                    result_string = 'ok';
                    break;
                }

                let this_ok_without_param = iterEquals(requotedGold[referenceId], requoteProgram(normalized));
                beam_ok_without_param = beam_ok_without_param || this_ok_without_param;
                if (this_ok_without_param && !beam_ok)
                    result_string = 'ok_without_param';

                let functions = Array.from(getFunctions(normalized));
                let this_ok_function = this_ok_without_param || iterEquals(goldFunctions[referenceId], functions);
                beam_ok_function = beam_ok_function || this_ok_function;
                if (this_ok_function && !beam_ok_without_param)
                    result_string = 'ok_function';

                let this_ok_device = this_ok_function || iterEquals(goldDevices[referenceId], getDevices(normalized));
                beam_ok_device = beam_ok_device || this_ok_device;
                if (this_ok_device && !beam_ok_function)
                    result_string = 'ok_device';

                let this_ok_num_function = this_ok_device || goldFunctions[referenceId].length === functions.length;
                beam_ok_num_function = beam_ok_num_function || this_ok_num_function;
                if (this_ok_num_function && !beam_ok_device)
                    result_string = 'ok_num_function';
            }

            if (first && this._debug && result_string !== 'ok')
                console.log(`${this._id}\t${result_string}\t${this._preprocessed}\t${target}\t${prediction}`);
            first = false;
            ok = ok || beam_ok;
            ok_without_param = ok_without_param || beam_ok_without_param;
            ok_function = ok_function || beam_ok_function;
            ok_device = ok_device || beam_ok_device;
            ok_num_function = ok_num_function || beam_ok_num_function;

            result.ok.push(ok);
            result.ok_without_param.push(ok_without_param);
            result.ok_function.push(ok_function);
            result.ok_device.push(ok_device);
            result.ok_num_function.push(ok_num_function);
            result.ok_syntax.push(ok_syntax);
        }

        return result;
    }
}

class SentenceEvaluatorStream extends Stream.Transform {
    constructor(locale, parser, schemas, tokenized, debug, complexityMetric) {
        super({ objectMode: true });

        this._locale = locale;
        this._parser = parser;
        this._schemas = schemas;
        this._tokenized = tokenized;
        this._debug = debug;
        this._complexityMetric = complexityMetric;
    }

    _transform(ex, encoding, callback) {
        const evaluator = new SentenceEvaluator(this._locale, this._parser, this._schemas, this._tokenized, this._debug, this._complexityMetric, ex);

        evaluator.evaluate().then((result) => callback(null, result), (err) => callback(err));
    }

    _flush(callback) {
        process.nextTick(callback);
    }
}

class CollectSentenceStatistics extends Stream.Writable {
    constructor(options = {}) {
        super({ objectMode: true });

        this._minComplexity = options.minComplexity;
        this._maxComplexity = options.maxComplexity;
        this._splitByDevice = options.splitByDevice;

        // _buffer will map devices to individual buffers. If splitByDevice is
        // false, there will only be a single buffer. Otherwise, these individual
        // buffers will be created ad-hoc, as new devices come up.
        this._buffer = {};
    }

    _write(ex, encoding, callback) {
        let uniqueDevices;
        if (this._splitByDevice) {
            uniqueDevices = Array.from(new Set(ex.target_devices));
            if (uniqueDevices.length === 0)
                uniqueDevices.push('generic'); // generic device, for deviceless acts
        } else {
            uniqueDevices = ['all_devices'];
        }
        for (let device of uniqueDevices) {
            if (!(device in this._buffer)) {
                this._buffer[device] = {
                    total: 0,
                    primitives: 0,
                    compounds: 0,
                    ok: [],
                    ok_without_param: [],
                    ok_function: [],
                    ok_device: [],
                    ok_num_function: [],
                    ok_syntax: [],
                    'prim/ok': [],
                    'prim/ok_without_param': [],
                    'prim/ok_function': [],
                    'prim/ok_device': [],
                    'prim/ok_num_function': [],
                    'prim/ok_syntax': [],
                    'comp/ok': [],
                    'comp/ok_without_param': [],
                    'comp/ok_function': [],
                    'comp/ok_device': [],
                    'comp/ok_num_function': [],
                    'comp/ok_syntax': [],
                };
            }
            this._buffer[device].total ++;
            if (ex.is_primitive)
                this._buffer[device].primitives ++;
            else
                this._buffer[device].compounds ++;

            let compkey;
            if (this._minComplexity > 0 && ex.complexity <= this._minComplexity)
                compkey = 'complexity_<=' + this._minComplexity + '/';
            else if (this._maxComplexity && ex.complexity >= this._maxComplexity)
                compkey = 'complexity_>=' + this._maxComplexity + '/';
            else
                compkey = 'complexity_' + ex.complexity + '/';
            if (!this._buffer[device][compkey + 'total']) {
                this._buffer[device][compkey + 'total'] = 0;
                for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax'])
                    this._buffer[device][compkey + key] = [];
            }


            let numericKey = ex.has_numeric ? 'with_numeric_' : 'without_numeric_';
            if (!this._buffer[device][numericKey + 'total']) {
                this._buffer[device][numericKey + 'total'] = 0;
                for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax'])
                    this._buffer[device][numericKey + key] = [];
            }

            this._buffer[device][compkey + 'total'] += 1;
            this._buffer[device][numericKey + 'total'] += 1;

            for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {
                for (let beampos = 0; beampos < ex[key].length; beampos++) {
                    while (this._buffer[device][key].length <= beampos)
                        this._buffer[device][key].push(0);
                    if (ex[key][beampos])
                        this._buffer[device][key][beampos] ++;

                    let subkey = ex.is_primitive ? 'prim/' + key : 'comp/' + key;
                    while (this._buffer[device][subkey].length <= beampos)
                        this._buffer[device][subkey].push(0);
                    if (ex[key][beampos])
                        this._buffer[device][subkey][beampos] ++;

                    subkey = compkey + key;
                    while (this._buffer[device][subkey].length <= beampos)
                        this._buffer[device][subkey].push(0);
                    if (ex[key][beampos])
                        this._buffer[device][subkey][beampos] ++;
                    assert(!isNaN(this._buffer[device][subkey][beampos]));

                    subkey = numericKey + key;
                    while (this._buffer[device][subkey].length <= beampos)
                        this._buffer[device][subkey].push(0);
                    if (ex[key][beampos])
                        this._buffer[device][subkey][beampos] ++;
                }
            }
        }
        callback();
    }

    _final(callback) {
        for (let device in this._buffer) {
            // convert to percentages
            for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {
                for (let beampos = 0; beampos < this._buffer[device][key].length; beampos++) {
                    //this._buffer[device][key][beampos] = (this._buffer[device][key][beampos] * 100 / this._buffer[device].total).toFixed(2);
                    //this._buffer[device]['prim/' + key][beampos] = (this._buffer[device]['prim/' + key][beampos] * 100 / this._buffer[device].primitives).toFixed(2);
                    //this._buffer[device]['comp/' + key][beampos] = (this._buffer[device]['comp/' + key][beampos] * 100 / this._buffer[device].compounds).toFixed(2);

                    this._buffer[device][key][beampos] /= this._buffer[device].total;
                    this._buffer[device]['prim/' + key][beampos] /= this._buffer[device].primitives;
                    this._buffer[device]['comp/' + key][beampos] /= this._buffer[device].compounds;

                    let compkey = this._minComplexity > 0 ? 'complexity_<=' + this._minComplexity + '/' : 'complexity_0/';
                    if (this._buffer[device][compkey + 'total']) {
                        this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];
                        assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);
                    }
                    for (let i = this._minComplexity; i < 20; i++) {
                        compkey = 'complexity_' + i + '/';
                        if (this._buffer[device][compkey + 'total']) {
                            this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];
                            assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);
                        }
                    }
                    if (this._maxComplexity) {
                        compkey = 'complexity_>=' + this._maxComplexity + '/';
                        if (this._buffer[device][compkey + 'total']) {
                            this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];
                            assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);
                        }
                    }

                    let numerickey = 'with_numeric_';
                    if (this._buffer[device][numerickey + 'total']) {
                        this._buffer[device][numerickey + key][beampos] /= this._buffer[device][numerickey + 'total'];
                        assert(!isNaN(this._buffer[device][numerickey + key][beampos]), this._buffer[device][numerickey + key]);
                    }

                    numerickey = 'without_numeric_';
                    if (this._buffer[device][numerickey + 'total']) {
                        this._buffer[device][numerickey + key][beampos] /= this._buffer[device][numerickey + 'total'];
                        assert(!isNaN(this._buffer[device][numerickey + key][beampos]), this._buffer[device][numerickey + key]);
                    }
                }
            }
        }
        callback();
    }

    read() {
        return new Promise((resolve, reject) => {
            this.on('finish', () => resolve(this._buffer));
            this.on('error', reject);
        });
    }
}

module.exports = {
    SentenceEvaluatorStream,
    CollectSentenceStatistics,
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const assert = require('assert');	56✔
23	const Stream = require('stream');	56✔
24	const ThingTalk = require('thingtalk');	56✔
25
26	const Utils = require('../../utils/misc-utils');	56✔
27	const { requoteProgram, getFunctions, getDevices } = require('../requoting');	56✔
28	const { stripOutTypeAnnotations, normalizeKeywordParams } = require('./eval_utils');	56✔
29	const I18n = require('../../i18n');	56✔
30
31	function iterEquals(iterable1, iterable2) {
32	let iter1 = iterable1[Symbol.iterator]();	×
33	let iter2 = iterable2[Symbol.iterator]();	×
34	for (;;) {	×
35	let { value: value1, done: done1 } = iter1.next();	×
36	let { value: value2, done: done2 } = iter2.next();	×
37	if (done1 !== done2)	×
38	return false;	×
39	if (done1)	×
40	break;	×
41	if (value1 !== value2)	×
42	return false;	×
43	}
44	return true;	×
45	}
46
47	const COMPLEXITY_METRICS = {	56✔
48	num_params(id, code) {
49	let params = 0;	1✔
50	let joins = 0;	1✔
51	let inString = false;	1✔
52	for (let token of code.split(' ')) {	1✔
53	if (token === '"')	5!
54	inString = !inString;	×
55	if (inString)	5!
56	continue;	×
57
58	// distance is computed by geo, which will be counted
59	if (token.startsWith('param:') && !token.startsWith('param:distance'))	5!
60	params ++;	×
61	else if (token === 'join')	5!
62	joins ++;	×
63	}
64
65	return params + joins;	1✔
66	},
67
68	turn_number(id, code) {
69	const match = /\/([0-9]+)(?:-[0-9]+)*$/.exec(id);	×
70	return parseInt(match[1]);	×
71	}
72	};
73
74
75	class SentenceEvaluator {
76	constructor(locale, parser, schemaRetriever, tokenized, debug, complexityMetric, ex) {
77	this._parser = parser;	1✔
78	this._tokenized = tokenized;	1✔
79	this._debug = debug;	1✔
80	this._schemas = schemaRetriever;	1✔
81	this._tokenizer = I18n.get(locale).getTokenizer();	1✔
82
83	this._id = ex.id;	1✔
84	this._context = ex.context;	1✔
85	this._preprocessed = ex.preprocessed;	1✔
86	this._targetPrograms = ex.target_code;	1✔
87	this._predictions = ex.predictions;	1✔
88
89	if (complexityMetric)	1!
90	this._computeComplexity = COMPLEXITY_METRICS[complexityMetric];	1✔
91	}
92
93	_hasNumeric(code) {
94	for (let token of code.split(' ')) {	1✔
95	if (!isNaN(token))	5!
96	return true;	×
97	if (token === '>=' \|\| token === '<=')	5!
98	return true;	×
99	if (token.startsWith('NUMBER_'))	5!
100	return true;	×
101	}
102	return false;	1✔
103	}
104
105	async evaluate() {
106
107	const result = {	1✔
108	id: this._id,
109	preprocessed: this._preprocessed,
110	target_code: this._targetPrograms,
111	target_devices: [],
112	ok: [],
113	ok_without_param: [],
114	ok_function: [],
115	ok_device: [],
116	ok_num_function: [],
117	ok_syntax: [],
118
119	is_primitive: false,
120	complexity: undefined,
121	has_numeric: false
122	};
123
124	let contextCode = undefined, contextEntities = {};	1✔
125	if (this._context !== undefined) {	1!
126	contextCode = this._context.split(' ');	×
127	contextEntities = Utils.makeDummyEntities(this._context);	×
128	}
129
130	let entities;
131	if (this._tokenized) {	1!
132	entities = Utils.makeDummyEntities(this._preprocessed);	1✔
133	Object.assign(entities, contextEntities);	1✔
134	} else {
135	const tokenized = await this._tokenizer.tokenize(this._preprocessed);	×
136	Utils.renumberEntities(tokenized, contextEntities);	×
137	entities = tokenized.entities;	×
138	}
139
140	assert(Array.isArray(this._targetPrograms));	1✔
141	assert(this._targetPrograms.length > 0);	1✔
142
143	const untypedTargetCode = [];	1✔
144	const normalEntities = {};	1✔
145	let firstTargetCode = this._targetPrograms[0];	1✔
146	try {	1✔
147	const sequence = firstTargetCode.split(' ');	1✔
148	const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);	1✔
149	await parsed.typecheck(this._schemas);	1✔
150	const normalized = ThingTalk.NNSyntax.toNN(parsed, this._preprocessed, normalEntities, {	1✔
151	allocateEntities: true,
152	typeAnnotations: false
153	});
154	untypedTargetCode.push(normalized.join(' '));	1✔
155	} catch(e) {
156	// if the target_code did not parse due to missing functions in thingpedia, ignore it
157	if (e.message.indexOf('has no query') >= 0 \|\| e.message.indexOf('has no action') >= 0)	×
158	return null;	×
159
160	console.error(this._id, this._preprocessed, this._targetPrograms);	×
161	throw e;	×
162	}
163	if (this._computeComplexity)	1!
164	result.complexity = this._computeComplexity(this._id, this._targetPrograms[0]);	1✔
165	else
166	result.complexity = 0;	×
167	result.has_numeric = this._hasNumeric(this._targetPrograms[0]);	1✔
168
169	// check all other target codes (sanity check)
170	for (let i = 1; i < this._targetPrograms.length; i++) {	1✔
171	try {	×
172	const sequence = this._targetPrograms[i].split(' ');	×
173	const parsed = ThingTalk.NNSyntax.fromNN(sequence, entities);	×
174	await parsed.typecheck(this._schemas);	×
175	const normalized = ThingTalk.NNSyntax.toNN(parsed, this._preprocessed, normalEntities, {	×
176	allocateEntities: true,
177	typeAnnotations: false
178	});
179	untypedTargetCode.push(normalized.join(' '));	×
180	} catch(e) {
181	console.error(this._id, this._preprocessed, this._targetPrograms);	×
182	throw e;	×
183	}
184	}
185
186	const requotedGold = untypedTargetCode.map((code) => Array.from(requoteProgram(code)));	1✔
187	const goldFunctions = untypedTargetCode.map((code) => Array.from(getFunctions(code)));	1✔
188	const goldDevices = untypedTargetCode.map((code) => Array.from(getDevices(code)));	1✔
189	result.is_primitive = goldFunctions[0].length === 1;	1✔
190	result.target_devices = goldDevices[0];	1✔
191
192	let first = true;	1✔
193	let ok = false, ok_without_param = false, ok_function = false,	1✔
194	ok_device = false, ok_num_function = false, ok_syntax = false;	1✔
195
196	let predictions;
197	if (this._predictions) {	1!
198	predictions = this._predictions;	×
199	} else {
200	try {	1✔
201	const parsed = await this._parser.sendUtterance(this._preprocessed, contextCode, contextEntities, {	1✔
202	tokenized: this._tokenized,
203	skip_typechecking: true
204	});
205	if (!entities)	1!
206	entities = parsed.entities;	×
207
208	predictions = parsed.candidates	1✔
209	.filter((beam) => beam.score !== 'Infinity') // ignore exact matches	1✔
210	.map((beam) => beam.code);	1✔
211	} catch(e) {
212	console.error(`Sentence ${this._id} failed to predict`);	×
213	console.error(e);	×
214	predictions = [[]];	×
215	}
216	}
217
218	for (let beam of predictions) {	1✔
219	let target = Array.from(stripOutTypeAnnotations(this._targetPrograms[0].split(' '))).join(' ');	1✔
220	let prediction = Array.from(stripOutTypeAnnotations(beam)).join(' ');	1✔
221
222	// first check if the program parses and typechecks (no hope otherwise)
223	try {	1✔
224	const parsed = ThingTalk.NNSyntax.fromNN(beam, entities);	1✔
225	await parsed.typecheck(this._schemas);	×
226	beam = ThingTalk.NNSyntax.toNN(parsed, beam, normalEntities, { allocateEntities: true });	×
227	} catch(e) {
228	// push the previous result, so the stats
229	// stay cumulative along the beam
230
231	result.ok.push(ok);	1✔
232	result.ok_without_param.push(ok_without_param);	1✔
233	result.ok_function.push(ok_function);	1✔
234	result.ok_device.push(ok_device);	1✔
235	result.ok_num_function.push(ok_num_function);	1✔
236	result.ok_syntax.push(ok_syntax);	1✔
237	if (first && this._debug)	1!
238	console.log(`${this._id}\twrong_syntax\t${this._preprocessed}\t${target}\t${prediction}`);	1✔
239	first = false;	1✔
240	continue;	1✔
241	}
242	ok_syntax = true;	×
243
244	const normalized = normalizeKeywordParams(Array.from(stripOutTypeAnnotations(beam)));	×
245	const code = normalized.join(' ');	×
246
247	let beam_ok = false, beam_ok_without_param = false, beam_ok_function = false,	×
248	beam_ok_device = false, beam_ok_num_function = false;	×
249	let result_string = 'ok_syntax';	×
250
251	for (let referenceId = 0; referenceId < this._targetPrograms.length; referenceId++) {	×
252	if (code === untypedTargetCode[referenceId]) {	×
253	// we have a match!
254
255	beam_ok = true;	×
256	beam_ok_without_param = true;	×
257	beam_ok_function = true;	×
258	beam_ok_device = true;	×
259	beam_ok_num_function = true;	×
260	result_string = 'ok';	×
261	break;	×
262	}
263
264	let this_ok_without_param = iterEquals(requotedGold[referenceId], requoteProgram(normalized));	×
265	beam_ok_without_param = beam_ok_without_param \|\| this_ok_without_param;	×
266	if (this_ok_without_param && !beam_ok)	×
267	result_string = 'ok_without_param';	×
268
269	let functions = Array.from(getFunctions(normalized));	×
270	let this_ok_function = this_ok_without_param \|\| iterEquals(goldFunctions[referenceId], functions);	×
271	beam_ok_function = beam_ok_function \|\| this_ok_function;	×
272	if (this_ok_function && !beam_ok_without_param)	×
273	result_string = 'ok_function';	×
274
275	let this_ok_device = this_ok_function \|\| iterEquals(goldDevices[referenceId], getDevices(normalized));	×
276	beam_ok_device = beam_ok_device \|\| this_ok_device;	×
277	if (this_ok_device && !beam_ok_function)	×
278	result_string = 'ok_device';	×
279
280	let this_ok_num_function = this_ok_device \|\| goldFunctions[referenceId].length === functions.length;	×
281	beam_ok_num_function = beam_ok_num_function \|\| this_ok_num_function;	×
282	if (this_ok_num_function && !beam_ok_device)	×
283	result_string = 'ok_num_function';	×
284	}
285
286	if (first && this._debug && result_string !== 'ok')	×
287	console.log(`${this._id}\t${result_string}\t${this._preprocessed}\t${target}\t${prediction}`);	×
288	first = false;	×
289	ok = ok \|\| beam_ok;	×
290	ok_without_param = ok_without_param \|\| beam_ok_without_param;	×
291	ok_function = ok_function \|\| beam_ok_function;	×
292	ok_device = ok_device \|\| beam_ok_device;	×
293	ok_num_function = ok_num_function \|\| beam_ok_num_function;	×
294
295	result.ok.push(ok);	×
296	result.ok_without_param.push(ok_without_param);	×
297	result.ok_function.push(ok_function);	×
298	result.ok_device.push(ok_device);	×
299	result.ok_num_function.push(ok_num_function);	×
300	result.ok_syntax.push(ok_syntax);	×
301	}
302
303	return result;	1✔
304	}
305	}
306
307	class SentenceEvaluatorStream extends Stream.Transform {
308	constructor(locale, parser, schemas, tokenized, debug, complexityMetric) {
309	super({ objectMode: true });	1✔
310
311	this._locale = locale;	1✔
312	this._parser = parser;	1✔
313	this._schemas = schemas;	1✔
314	this._tokenized = tokenized;	1✔
315	this._debug = debug;	1✔
316	this._complexityMetric = complexityMetric;	1✔
317	}
318
319	_transform(ex, encoding, callback) {
320	const evaluator = new SentenceEvaluator(this._locale, this._parser, this._schemas, this._tokenized, this._debug, this._complexityMetric, ex);	1✔
321
322	evaluator.evaluate().then((result) => callback(null, result), (err) => callback(err));	1✔
323	}
324
325	_flush(callback) {
326	process.nextTick(callback);	1✔
327	}
328	}
329
330	class CollectSentenceStatistics extends Stream.Writable {
331	constructor(options = {}) {	×
332	super({ objectMode: true });	1✔
333
334	this._minComplexity = options.minComplexity;	1✔
335	this._maxComplexity = options.maxComplexity;	1✔
336	this._splitByDevice = options.splitByDevice;	1✔
337
338	// _buffer will map devices to individual buffers. If splitByDevice is
339	// false, there will only be a single buffer. Otherwise, these individual
340	// buffers will be created ad-hoc, as new devices come up.
341	this._buffer = {};	1✔
342	}
343
344	_write(ex, encoding, callback) {
345	let uniqueDevices;
346	if (this._splitByDevice) {	1!
347	uniqueDevices = Array.from(new Set(ex.target_devices));	×
348	if (uniqueDevices.length === 0)	×
349	uniqueDevices.push('generic'); // generic device, for deviceless acts	×
350	} else {
351	uniqueDevices = ['all_devices'];	1✔
352	}
353	for (let device of uniqueDevices) {	1✔
354	if (!(device in this._buffer)) {	1!
355	this._buffer[device] = {	1✔
356	total: 0,
357	primitives: 0,
358	compounds: 0,
359	ok: [],
360	ok_without_param: [],
361	ok_function: [],
362	ok_device: [],
363	ok_num_function: [],
364	ok_syntax: [],
365	'prim/ok': [],
366	'prim/ok_without_param': [],
367	'prim/ok_function': [],
368	'prim/ok_device': [],
369	'prim/ok_num_function': [],
370	'prim/ok_syntax': [],
371	'comp/ok': [],
372	'comp/ok_without_param': [],
373	'comp/ok_function': [],
374	'comp/ok_device': [],
375	'comp/ok_num_function': [],
376	'comp/ok_syntax': [],
377	};
378	}
379	this._buffer[device].total ++;	1✔
380	if (ex.is_primitive)	1!
381	this._buffer[device].primitives ++;	1✔
382	else
383	this._buffer[device].compounds ++;	×
384
385	let compkey;
386	if (this._minComplexity > 0 && ex.complexity <= this._minComplexity)	1!
NEW 387	compkey = 'complexity_<=' + this._minComplexity + '/';	×
388	else if (this._maxComplexity && ex.complexity >= this._maxComplexity)	1!
UNCOV 389	compkey = 'complexity_>=' + this._maxComplexity + '/';	×
390	else
391	compkey = 'complexity_' + ex.complexity + '/';	1✔
392	if (!this._buffer[device][compkey + 'total']) {	1!
393	this._buffer[device][compkey + 'total'] = 0;	1✔
394	for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax'])	1✔
395	this._buffer[device][compkey + key] = [];	6✔
396	}
397
398
399	let numericKey = ex.has_numeric ? 'with_numeric_' : 'without_numeric_';	1!
400	if (!this._buffer[device][numericKey + 'total']) {	1!
401	this._buffer[device][numericKey + 'total'] = 0;	1✔
402	for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax'])	1✔
403	this._buffer[device][numericKey + key] = [];	6✔
404	}
405
406	this._buffer[device][compkey + 'total'] += 1;	1✔
407	this._buffer[device][numericKey + 'total'] += 1;	1✔
408
409	for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {	1✔
410	for (let beampos = 0; beampos < ex[key].length; beampos++) {	6✔
411	while (this._buffer[device][key].length <= beampos)	6✔
412	this._buffer[device][key].push(0);	6✔
413	if (ex[key][beampos])	6!
414	this._buffer[device][key][beampos] ++;	×
415
416	let subkey = ex.is_primitive ? 'prim/' + key : 'comp/' + key;	6!
417	while (this._buffer[device][subkey].length <= beampos)	6✔
418	this._buffer[device][subkey].push(0);	6✔
419	if (ex[key][beampos])	6!
420	this._buffer[device][subkey][beampos] ++;	×
421
422	subkey = compkey + key;	6✔
423	while (this._buffer[device][subkey].length <= beampos)	6✔
424	this._buffer[device][subkey].push(0);	6✔
425	if (ex[key][beampos])	6!
426	this._buffer[device][subkey][beampos] ++;	×
427	assert(!isNaN(this._buffer[device][subkey][beampos]));	6✔
428
429	subkey = numericKey + key;	6✔
430	while (this._buffer[device][subkey].length <= beampos)	6✔
431	this._buffer[device][subkey].push(0);	6✔
432	if (ex[key][beampos])	6!
433	this._buffer[device][subkey][beampos] ++;	×
434	}
435	}
436	}
437	callback();	1✔
438	}
439
440	_final(callback) {
441	for (let device in this._buffer) {	1✔
442	// convert to percentages
443	for (let key of ['ok', 'ok_without_param', 'ok_function', 'ok_device', 'ok_num_function', 'ok_syntax']) {	1✔
444	for (let beampos = 0; beampos < this._buffer[device][key].length; beampos++) {	6✔
445	//this._buffer[device][key][beampos] = (this._buffer[device][key][beampos] * 100 / this._buffer[device].total).toFixed(2);
446	//this._buffer[device]['prim/' + key][beampos] = (this._buffer[device]['prim/' + key][beampos] * 100 / this._buffer[device].primitives).toFixed(2);
447	//this._buffer[device]['comp/' + key][beampos] = (this._buffer[device]['comp/' + key][beampos] * 100 / this._buffer[device].compounds).toFixed(2);
448
449	this._buffer[device][key][beampos] /= this._buffer[device].total;	6✔
450	this._buffer[device]['prim/' + key][beampos] /= this._buffer[device].primitives;	6✔
451	this._buffer[device]['comp/' + key][beampos] /= this._buffer[device].compounds;	6✔
452
453	let compkey = this._minComplexity > 0 ? 'complexity_<=' + this._minComplexity + '/' : 'complexity_0/';	6!
454	if (this._buffer[device][compkey + 'total']) {	6!
455	this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];	6✔
456	assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);	6✔
457	}
458	for (let i = this._minComplexity; i < 20; i++) {	6✔
459	compkey = 'complexity_' + i + '/';	120✔
460	if (this._buffer[device][compkey + 'total']) {	120✔
461	this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];	6✔
462	assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);	6✔
463	}
464	}
465	if (this._maxComplexity) {	6!
466	compkey = 'complexity_>=' + this._maxComplexity + '/';	6✔
467	if (this._buffer[device][compkey + 'total']) {	6!
468	this._buffer[device][compkey + key][beampos] /= this._buffer[device][compkey + 'total'];	×
469	assert(!isNaN(this._buffer[device][compkey + key][beampos]), this._buffer[device][compkey + key]);	×
470	}
471	}
472
473	let numerickey = 'with_numeric_';	6✔
474	if (this._buffer[device][numerickey + 'total']) {	6!
475	this._buffer[device][numerickey + key][beampos] /= this._buffer[device][numerickey + 'total'];	×
476	assert(!isNaN(this._buffer[device][numerickey + key][beampos]), this._buffer[device][numerickey + key]);	×
477	}
478
479	numerickey = 'without_numeric_';	6✔
480	if (this._buffer[device][numerickey + 'total']) {	6!
481	this._buffer[device][numerickey + key][beampos] /= this._buffer[device][numerickey + 'total'];	6✔
482	assert(!isNaN(this._buffer[device][numerickey + key][beampos]), this._buffer[device][numerickey + key]);	6✔
483	}
484	}
485	}
486	}
487	callback();	1✔
488	}
489
490	read() {
491	return new Promise((resolve, reject) => {	1✔
492	this.on('finish', () => resolve(this._buffer));	1✔
493	this.on('error', reject);	1✔
494	});
495	}
496	}
497
498	module.exports = {	56✔
499	SentenceEvaluatorStream,
500	CollectSentenceStatistics,
501	};

stanford-oval / genie-toolkit / 1232

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous