stanford-oval / genie-toolkit / 630

pending completion

Build # 630

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Merge pull request #134 from stanford-oval/wip/thingpedia

Bug fixes for Thingpedia

Run Details

1731 of 4470 branches covered (38.72%)

Branch coverage included in aggregate %.

1 of 36 new or added lines in 4 files covered. (2.78%)

2 existing lines in 1 file now uncovered.

4353 of 9100 relevant lines covered (47.84%)

563730.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

15.38

/tool/lib/parserclient.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Almond
//
// Copyright 2016 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const ThingTalk = require('thingtalk');
const Tp = require('thingpedia');
const qs = require('qs');

const TokenizerService = require('../../lib/tokenizer');
const Predictor = require('../../lib/predictor');
const Utils = require('../../lib/utils');
const I18n = require('../../lib/i18n');

const SEMANTIC_PARSING_TASK = 'almond';
const NLU_TASK = 'almond_dialogue_nlu';
const NLG_TASK = 'almond_dialogue_nlg';
const NLG_QUESTION = 'what should the agent say ?';
const POLICY_QUESTION = 'what should the agent do ?';
const POLICY_TASK = 'almond_dialogue_policy';

class LocalParserClient {
    constructor(modeldir, locale) {
        this._locale = locale;
        this._tokenizer = TokenizerService.get('local');
        this._predictor = new Predictor('local', modeldir);
        this._langPack = I18n.get(locale);
    }

    async start() {
        await this._predictor.start();
    }
    async stop() {
        await this._predictor.stop();
        await this._tokenizer.end();
    }

    async tokenize(utterance, contextEntities) {
        const tokenized = await this._tokenizer.tokenize(this._locale, utterance);
        Utils.renumberEntities(tokenized, contextEntities);
        return tokenized;

    }
    async sendUtterance(utterance, tokenized, contextCode, contextEntities) {
        let tokens, entities;
        if (tokenized) {
            tokens = utterance.split(' ');
            entities = Utils.makeDummyEntities(utterance);
            Object.assign(entities, contextEntities);
        } else {
            const tokenized = await this._tokenizer.tokenize(this._locale, utterance);
            Utils.renumberEntities(tokenized, contextEntities);
            tokens = tokenized.tokens;
            entities = tokenized.entities;
        }

        let candidates;
        if (contextCode)
            candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), NLU_TASK);
        else
            candidates = await this._predictor.predict(tokens.join(' '), undefined, SEMANTIC_PARSING_TASK);

        candidates = candidates.map((cand) => {
            return {
                code: cand.answer.split(' '),
                score: cand.score
            };
        });
        return { tokens, candidates, entities };
    }
    async queryPolicy(contextCode, contextEntities) {
        return this._predictor.predict(contextCode.join(' '), POLICY_QUESTION, POLICY_TASK);
    }
    async generateUtterance(contextCode, contextEntities, targetAct) {
        let candidates = this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, NLG_TASK);
        candidates = candidates.map((cand) => {
            return {
                answer: this._langPack.postprocessNLG(cand.answer, contextEntities),
                score: cand.score
            };
        });
        return candidates;
    }
}

class RemoteParserClient {
    constructor(baseUrl, locale) {
        this._locale = locale;
        this._baseUrl = baseUrl + '/' + this._locale;
    }

    async start() {}
    async stop() {}

    async tokenize(utterance, contextEntities) {
        const data = {
            q: utterance,
        };

        let response;
        if (contextEntities !== undefined) {
            data.entities = contextEntities;

            response = await Tp.Helpers.Http.post(`${this._baseUrl}/tokenize`, JSON.stringify(data), {
                dataContentType: 'application/json'
            });
        } else {
            let url = `${this._baseUrl}/tokenize?${qs.stringify(data)}`;

            response = await Tp.Helpers.Http.get(url);
        }
        const parsed = JSON.parse(response);

        if (parsed.error)
            throw new Error('Error received from Genie-Parser server: ' + parsed.error);

        return parsed;
    }

    async sendUtterance(utterance, tokenized, contextCode, contextEntities) {
        const data = {
            q: utterance,
            store: 'no',
            thingtalk_version: ThingTalk.version,
        };

        if (contextCode !== undefined) {
            data.context = contextCode.join(' ');
            data.entities = contextEntities;
            data.tokenized = tokenized;
            data.skip_typechecking = true;
        }
        const response = await Tp.Helpers.Http.post(`${this._baseUrl}/query`, JSON.stringify(data), {
            dataContentType: 'application/json'
        });
        const parsed = JSON.parse(response);
        if (parsed.error)
            throw new Error('Error received from Genie server: ' + parsed.error);

        return parsed;
    }

    async generateUtterance(contextCode, contextEntities, targetAct) {
        const data = {
            context: contextCode.join(' '),
            entities: contextEntities,
            target: targetAct.join(' ')
        };
        const response = await Tp.Helpers.Http.post(`${this._baseUrl}/answer`, JSON.stringify(data), {
            dataContentType: 'application/json' //'
        });
        const parsed = JSON.parse(response);
        if (parsed.error)
            throw new Error('Error received from Genie server: ' + parsed.error);

        return parsed;
    }
}

module.exports = {
    get(url, locale) {
        if (url.startsWith('file://'))
            return new LocalParserClient(url.substring('file://'.length), locale);
        else
            return new RemoteParserClient(url, locale);
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Almond
4	//
5	// Copyright 2016 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const ThingTalk = require('thingtalk');	21✔
13	const Tp = require('thingpedia');	21✔
14	const qs = require('qs');	21✔
15
16	const TokenizerService = require('../../lib/tokenizer');	21✔
17	const Predictor = require('../../lib/predictor');	21✔
18	const Utils = require('../../lib/utils');	21✔
19	const I18n = require('../../lib/i18n');	21✔
20
21	const SEMANTIC_PARSING_TASK = 'almond';	21✔
22	const NLU_TASK = 'almond_dialogue_nlu';	21✔
23	const NLG_TASK = 'almond_dialogue_nlg';	21✔
24	const NLG_QUESTION = 'what should the agent say ?';	21✔
25	const POLICY_QUESTION = 'what should the agent do ?';	21✔
26	const POLICY_TASK = 'almond_dialogue_policy';	21✔
27
28	class LocalParserClient {
29	constructor(modeldir, locale) {
30	this._locale = locale;	×
31	this._tokenizer = TokenizerService.get('local');	×
32	this._predictor = new Predictor('local', modeldir);	×
33	this._langPack = I18n.get(locale);	×
34	}
35
36	async start() {
37	await this._predictor.start();	×
38	}
39	async stop() {
40	await this._predictor.stop();	×
41	await this._tokenizer.end();	×
42	}
43
44	async tokenize(utterance, contextEntities) {
45	const tokenized = await this._tokenizer.tokenize(this._locale, utterance);	×
46	Utils.renumberEntities(tokenized, contextEntities);	×
47	return tokenized;	×
48
49	}
50	async sendUtterance(utterance, tokenized, contextCode, contextEntities) {
51	let tokens, entities;
52	if (tokenized) {	×
53	tokens = utterance.split(' ');	×
54	entities = Utils.makeDummyEntities(utterance);	×
55	Object.assign(entities, contextEntities);	×
56	} else {
57	const tokenized = await this._tokenizer.tokenize(this._locale, utterance);	×
58	Utils.renumberEntities(tokenized, contextEntities);	×
59	tokens = tokenized.tokens;	×
60	entities = tokenized.entities;	×
61	}
62
63	let candidates;
NEW 64	if (contextCode)	×
NEW 65	candidates = await this._predictor.predict(contextCode.join(' '), tokens.join(' '), NLU_TASK);	×
66	else
NEW 67	candidates = await this._predictor.predict(tokens.join(' '), undefined, SEMANTIC_PARSING_TASK);	×
68
69	candidates = candidates.map((cand) => {	×
70	return {	×
71	code: cand.answer.split(' '),
72	score: cand.score
73	};
74	});
75	return { tokens, candidates, entities };	×
76	}
77	async queryPolicy(contextCode, contextEntities) {
78	return this._predictor.predict(contextCode.join(' '), POLICY_QUESTION, POLICY_TASK);	×
79	}
80	async generateUtterance(contextCode, contextEntities, targetAct) {
81	let candidates = this._predictor.predict(contextCode.join(' ') + ' ' + targetAct.join(' '), NLG_QUESTION, NLG_TASK);	×
82	candidates = candidates.map((cand) => {	×
83	return {	×
84	answer: this._langPack.postprocessNLG(cand.answer, contextEntities),
85	score: cand.score
86	};
87	});
88	return candidates;	×
89	}
90	}
91
92	class RemoteParserClient {
93	constructor(baseUrl, locale) {
94	this._locale = locale;	×
95	this._baseUrl = baseUrl + '/' + this._locale;	×
96	}
97
98	async start() {}
99	async stop() {}
100
101	async tokenize(utterance, contextEntities) {
102	const data = {	×
103	q: utterance,
104	};
105
106	let response;
107	if (contextEntities !== undefined) {	×
108	data.entities = contextEntities;	×
109
110	response = await Tp.Helpers.Http.post(`${this._baseUrl}/tokenize`, JSON.stringify(data), {	×
111	dataContentType: 'application/json'
112	});
113	} else {
114	let url = `${this._baseUrl}/tokenize?${qs.stringify(data)}`;	×
115
116	response = await Tp.Helpers.Http.get(url);	×
117	}
118	const parsed = JSON.parse(response);	×
119
120	if (parsed.error)	×
121	throw new Error('Error received from Genie-Parser server: ' + parsed.error);	×
122
123	return parsed;	×
124	}
125
126	async sendUtterance(utterance, tokenized, contextCode, contextEntities) {
127	const data = {	×
128	q: utterance,
129	store: 'no',
130	thingtalk_version: ThingTalk.version,
131	};
132
133	if (contextCode !== undefined) {	×
134	data.context = contextCode.join(' ');	×
135	data.entities = contextEntities;	×
136	data.tokenized = tokenized;	×
137	data.skip_typechecking = true;	×
138	}
139	const response = await Tp.Helpers.Http.post(`${this._baseUrl}/query`, JSON.stringify(data), {	×
140	dataContentType: 'application/json'
141	});
142	const parsed = JSON.parse(response);	×
143	if (parsed.error)	×
144	throw new Error('Error received from Genie server: ' + parsed.error);	×
145
146	return parsed;	×
147	}
148
149	async generateUtterance(contextCode, contextEntities, targetAct) {
150	const data = {	×
151	context: contextCode.join(' '),
152	entities: contextEntities,
153	target: targetAct.join(' ')
154	};
155	const response = await Tp.Helpers.Http.post(`${this._baseUrl}/answer`, JSON.stringify(data), {	×
156	dataContentType: 'application/json' //'
157	});
158	const parsed = JSON.parse(response);	×
159	if (parsed.error)	×
160	throw new Error('Error received from Genie server: ' + parsed.error);	×
161
162	return parsed;	×
163	}
164	}
165
166	module.exports = {	21✔
167	get(url, locale) {
168	if (url.startsWith('file://'))	×
169	return new LocalParserClient(url.substring('file://'.length), locale);	×
170	else
171	return new RemoteParserClient(url, locale);	×
172	}
173	};

stanford-oval / genie-toolkit / 630

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous