stanford-oval / genie-toolkit / 885

pending completion

Build # 885

Build Type

Pull #178

travis-ci-com

Committed by

web-flow

Commit Message

Merge e3fb7ae8c into 82f4140fa

Pull Request Pull Request #178: Rewrite the tokenizer, kill almond-tokenizer

Run Details

1500 of 6343 branches covered (23.65%)

Branch coverage included in aggregate %.

762 of 839 new or added lines in 20 files covered. (90.82%)

212 existing lines in 3 files now uncovered.

4867 of 12500 relevant lines covered (38.94%)

112316.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.29

/tool/server.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const express = require('express');
const bodyParser = require('body-parser');
const logger = require('morgan');
const errorhandler = require('errorhandler');
const qv = require('query-validation');
const Tp = require('thingpedia');
const ThingTalk = require('thingtalk');

const Utils = require('../lib/utils');
const Predictor = require('../lib/predictor');
const I18n = require('../lib/i18n');

function learn(req, res) {
    res.status(501).json({ error: 'Learning is not available with this Genie server' });
}

const SEMANTIC_PARSING_TASK = 'almond';
const NLU_TASK = 'almond_dialogue_nlu';
const NLG_TASK = 'almond_dialogue_nlg';
const NLG_QUESTION = 'what should the agent say ?';

async function tokenize(params, data, res) {
    const app = res.app;
    if (params.locale !== app.args.locale) {
        res.status(404).json({ error: 'Unsupported language' });
        return;
    }

    const tokenized = await app.backend.tokenizer.tokenize(data.q, data.expect || null);
    if (data.entities)
        Utils.renumberEntities(tokenized, data.entities);

    res.cacheFor(3600);
    res.json(tokenized);
}

async function runNLUPrediction(backend, tokens, entities, context, limit, skipTypechecking) {
    let candidates;
    if (context === undefined)
        candidates = await backend.nlu.predict(tokens.join(' '), undefined, SEMANTIC_PARSING_TASK);
    else
        candidates = await backend.nlu.predict(context.join(' '), tokens.join(' '), NLU_TASK);

    // HACK:
    for (let cand of candidates) {
        if (cand.answer === '$dialogue @org.thingpedia.dialogue.transaction.execute ; now => [ param:reviewCount ] of ( ( result ( @com.yelp.restaurant param:reviewCount ] ) ) filter param:id == " golden boy pizza " ^^com.yelp:restaurant ) => param:price:Enum(cheap,moderate,expensive,luxury) ^^com.yelp:restaurant ) => notify ;')
            cand.answer = '$dialogue @org.thingpedia.dialogue.transaction.execute ; now => [ param:reviewCount ] of ( ( result ( @com.yelp.restaurant [ 1 ] ) ) filter param:id == " golden boy pizza " ^^com.yelp:restaurant ) => notify ;';

        cand.answer = cand.answer.replace(/param:(reviewCount|rating) == enum:/g, 'param:price == enum:');
    }
    console.log(candidates);

    if (skipTypechecking) {
        return candidates.map((c) => {
            return {
                code: c.answer.split(' '),
                score: c.score
            };
       }).slice(0, limit);
    }

    candidates = await Promise.all(candidates.map(async (c) => {
        try {
            const parsed = ThingTalk.NNSyntax.fromNN(c.answer.split(' '), entities);
            await parsed.typecheck(backend.schemas);
            return {
                code: c.answer.split(' '),
                score: c.score
            };
        } catch(e) {
            console.error(e);
            return null;
        }
    }));

    candidates = candidates.filter((c) => c !== null);

    if (limit >= 0)
        return candidates.slice(0, limit);
    else
        return candidates;
}

async function queryNLU(params, data, res) {
    const query = data.q;
    const thingtalk_version = data.thingtalk_version;
    const expect = data.expect || null;
    const isTokenized = !!data.tokenized;
    const app = res.app;

    if (thingtalk_version !== ThingTalk.version) {
        res.status(400).json({ error: 'Invalid ThingTalk version' });
        return;
    }
    if (params.locale !== app.args.locale) {
        res.status(400).json({ error: 'Unsupported language' });
        return;
    }

    // emulate the frontend classifier for API compatibility
    const intent = {
        question: 0,
        command: 1,
        chatty: 0,
        other: 0
    };

    let tokenized;
    if (isTokenized) {
        tokenized = {
            tokens: query.split(' '),
            entities: {},
        };
        if (data.entities) {
            // safety against weird properties
            for (let key of Object.getOwnPropertyNames(data.entities)) {
                if (/^(.+)_([0-9]+)$/.test(key))
                    tokenized[key] = data.entities[key];
            }
        }
    } else {
        tokenized = await app.backend.tokenizer.tokenize(query, expect);
        if (data.entities)
            Utils.renumberEntities(tokenized, data.entities);
    }

    const tokens = tokenized.tokens;
    let result;
    if (tokens.length === 0) {
        result = [{
            code: ['bookkeeping', 'special', 'special:failed'],
            score: 'Infinity'
        }];
    } else {
        result = await runNLUPrediction(app.backend, tokens, tokenized.entities,
                                        data.context ? data.context.split(' ') : undefined,
                                        data.limit ? parseInt(data.limit) : 5,
                                        !!data.skip_typechecking);
    }

    res.json({
         candidates: result,
         tokens: tokens,
         entities: tokenized.entities,
         intent
    });
}

async function runNLGPrediction(backend, context, entities, targetAct, limit) {
    let candidates = await backend.nlg.predict(context + ' ' + targetAct, NLG_QUESTION, NLG_TASK);

    candidates = candidates.slice(0, limit);

    candidates = candidates.map((cand) => {
        cand.answer = backend.i18n.postprocessNLG(cand.answer, entities);
        return cand;
    });

    return candidates;
}

async function queryNLG(params, data, res) {
    const app = res.app;

    if (params.locale !== app.args.locale) {
        res.status(400).json({ error: 'Unsupported language' });
        return;
    }

    const result = await runNLGPrediction(app.backend, data.context, data.entities, data.target,
                                          data.limit ? parseInt(data.limit) : 5);

    res.json({
         candidates: result,
    });
}

const QUERY_PARAMS = {
    q: 'string',
    store: '?string',
    access_token: '?string',
    thingtalk_version: '?string',
    limit: '?integer',
    expect: '?string',
    choices: '?array',
    context: '?string',
    entities: '?object',
    tokenized: 'boolean',
    skip_typechecking: 'boolean',
    developer_key: '?string',
};
const NLG_PARAMS = {
    context: 'string',
    entities: 'object',
    target: 'string',
};

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.addParser('server', {
            addHelp: true,
            description: "Expose a Genie-compatible NLP API over HTTP."
        });
        parser.addArgument(['-p', '--port'], {
            required: false,
            help: "HTTP port to listen on",
            defaultValue: 8400,
        });
        parser.addArgument('--nlu-model', {
            required: true,
            help: "Path to the NLU model, pointing to a model directory.",
        });
        parser.addArgument('--nlg-model', {
            required: false,
            help: "Path to the NLU model, pointing to a model directory.",
        });
        parser.addArgument('--thingpedia', {
            required: true,
            help: 'Path to ThingTalk file containing class definitions.'
        });
        parser.addArgument(['-l', '--locale'], {
            required: false,
            defaultValue: 'en-US',
            help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
        });
        parser.addArgument('--debug', {
            nargs: 0,
            action: 'storeTrue',
            help: 'Enable debugging.',
            defaultValue: true
        });
        parser.addArgument('--no-debug', {
            nargs: 0,
            action: 'storeFalse',
            dest: 'debug',
            help: 'Disable debugging.',
        });
    },

    async execute(args) {
        const tpClient = new Tp.FileClient(args);
        const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);
        const app = express();

        const i18n = I18n.get(args.locale);
        app.backend = {
            schemas,
            i18n,
            tokenizer: i18n.getTokenizer(),
            nlu: new Predictor('nlu', args.nlu_model, 1)
        };
        app.backend.nlu.start();
        if (args.nlg_model && args.nlg_model !== args.nlu_model) {
            app.backend.nlg = new Predictor('nlg', args.nlg_model, 1);
            app.backend.nlg.start();
        } else {
            app.backend.nlg = app.backend.nlu;
        }

        app.args = args;

        app.set('port', args.port);
        app.use(bodyParser.json());
        app.use(bodyParser.urlencoded({ extended: true }));
        app.use(logger('dev'));

        app.use((req, res, next) => {
            res.set('Access-Control-Allow-Origin', '*');
            next();
        });

        app.post('/:locale/query', qv.validatePOST(QUERY_PARAMS, { accept: 'application/json', json: true }), (req, res, next) => {
            queryNLU(req.params, req.body, res).catch(next);
        });

        app.post('/:locale/answer', qv.validatePOST(NLG_PARAMS, { accept: 'application/json', json: true }), (req, res, next) => {
            queryNLG(req.params, req.body, res).catch(next);
        });

        app.post('/:locale/tokenize', qv.validatePOST({ q: 'string', entities: '?object' }, { accept: 'application/json', json: true }), (req, res, next) => {
            tokenize(req.params, req.body, res).catch(next);
        });

        app.post('/:locale/learn', learn);

        // if we get here, we have a 404 error
        app.use('/', (req, res) => {
            res.status(404).json({ error: 'Invalid endpoint' });
        });
        app.use(errorhandler());

        const server = app.listen(app.get('port'));

        await new Promise((resolve, reject) => {
            process.on('SIGINT', resolve);
            process.on('SIGTERM', resolve);
        });

        await app.backend.nlu.stop();
        if (app.backend.nlg !== app.backend.nlu)
            await app.backend.nlg.stop();
        server.close();
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const express = require('express');	52✔
13	const bodyParser = require('body-parser');	52✔
14	const logger = require('morgan');	52✔
15	const errorhandler = require('errorhandler');	52✔
16	const qv = require('query-validation');	52✔
17	const Tp = require('thingpedia');	52✔
18	const ThingTalk = require('thingtalk');	52✔
19
20	const Utils = require('../lib/utils');	52✔
21	const Predictor = require('../lib/predictor');	52✔
22	const I18n = require('../lib/i18n');	52✔
23
24	function learn(req, res) {
25	res.status(501).json({ error: 'Learning is not available with this Genie server' });	×
26	}
27
28	const SEMANTIC_PARSING_TASK = 'almond';	52✔
29	const NLU_TASK = 'almond_dialogue_nlu';	52✔
30	const NLG_TASK = 'almond_dialogue_nlg';	52✔
31	const NLG_QUESTION = 'what should the agent say ?';	52✔
32
33	async function tokenize(params, data, res) {
34	const app = res.app;	×
35	if (params.locale !== app.args.locale) {	×
36	res.status(404).json({ error: 'Unsupported language' });	×
37	return;	×
38	}
39
NEW 40	const tokenized = await app.backend.tokenizer.tokenize(data.q, data.expect \|\| null);	×
41	if (data.entities)	×
42	Utils.renumberEntities(tokenized, data.entities);	×
43
44	res.cacheFor(3600);	×
45	res.json(tokenized);	×
46	}
47
48	async function runNLUPrediction(backend, tokens, entities, context, limit, skipTypechecking) {
49	let candidates;
50	if (context === undefined)	×
51	candidates = await backend.nlu.predict(tokens.join(' '), undefined, SEMANTIC_PARSING_TASK);	×
52	else
53	candidates = await backend.nlu.predict(context.join(' '), tokens.join(' '), NLU_TASK);	×
54
55	// HACK:
56	for (let cand of candidates) {	×
57	if (cand.answer === '$dialogue @org.thingpedia.dialogue.transaction.execute ; now => [ param:reviewCount ] of ( ( result ( @com.yelp.restaurant param:reviewCount ] ) ) filter param:id == " golden boy pizza " ^^com.yelp:restaurant ) => param:price:Enum(cheap,moderate,expensive,luxury) ^^com.yelp:restaurant ) => notify ;')	×
58	cand.answer = '$dialogue @org.thingpedia.dialogue.transaction.execute ; now => [ param:reviewCount ] of ( ( result ( @com.yelp.restaurant [ 1 ] ) ) filter param:id == " golden boy pizza " ^^com.yelp:restaurant ) => notify ;';	×
59
60	cand.answer = cand.answer.replace(/param:(reviewCount\|rating) == enum:/g, 'param:price == enum:');	×
61	}
62	console.log(candidates);	×
63
64	if (skipTypechecking) {	×
65	return candidates.map((c) => {	×
66	return {	×
67	code: c.answer.split(' '),
68	score: c.score
69	};
70	}).slice(0, limit);
71	}
72
73	candidates = await Promise.all(candidates.map(async (c) => {	×
74	try {	×
75	const parsed = ThingTalk.NNSyntax.fromNN(c.answer.split(' '), entities);	×
76	await parsed.typecheck(backend.schemas);	×
77	return {	×
78	code: c.answer.split(' '),
79	score: c.score
80	};
81	} catch(e) {
82	console.error(e);	×
83	return null;	×
84	}
85	}));
86
87	candidates = candidates.filter((c) => c !== null);	×
88
89	if (limit >= 0)	×
90	return candidates.slice(0, limit);	×
91	else
92	return candidates;	×
93	}
94
95	async function queryNLU(params, data, res) {
96	const query = data.q;	×
97	const thingtalk_version = data.thingtalk_version;	×
98	const expect = data.expect \|\| null;	×
99	const isTokenized = !!data.tokenized;	×
100	const app = res.app;	×
101
102	if (thingtalk_version !== ThingTalk.version) {	×
103	res.status(400).json({ error: 'Invalid ThingTalk version' });	×
104	return;	×
105	}
106	if (params.locale !== app.args.locale) {	×
107	res.status(400).json({ error: 'Unsupported language' });	×
108	return;	×
109	}
110
111	// emulate the frontend classifier for API compatibility
112	const intent = {	×
113	question: 0,
114	command: 1,
115	chatty: 0,
116	other: 0
117	};
118
119	let tokenized;
120	if (isTokenized) {	×
121	tokenized = {	×
122	tokens: query.split(' '),
123	entities: {},
124	};
125	if (data.entities) {	×
126	// safety against weird properties
127	for (let key of Object.getOwnPropertyNames(data.entities)) {	×
128	if (/^(.+)_([0-9]+)$/.test(key))	×
129	tokenized[key] = data.entities[key];	×
130	}
131	}
132	} else {
NEW 133	tokenized = await app.backend.tokenizer.tokenize(query, expect);	×
134	if (data.entities)	×
135	Utils.renumberEntities(tokenized, data.entities);	×
136	}
137
138	const tokens = tokenized.tokens;	×
139	let result;
140	if (tokens.length === 0) {	×
141	result = [{	×
142	code: ['bookkeeping', 'special', 'special:failed'],
143	score: 'Infinity'
144	}];
145	} else {
146	result = await runNLUPrediction(app.backend, tokens, tokenized.entities,	×
147	data.context ? data.context.split(' ') : undefined,	×
148	data.limit ? parseInt(data.limit) : 5,	×
149	!!data.skip_typechecking);
150	}
151
152	res.json({	×
153	candidates: result,
154	tokens: tokens,
155	entities: tokenized.entities,
156	intent
157	});
158	}
159
160	async function runNLGPrediction(backend, context, entities, targetAct, limit) {
161	let candidates = await backend.nlg.predict(context + ' ' + targetAct, NLG_QUESTION, NLG_TASK);	×
162
163	candidates = candidates.slice(0, limit);	×
164
165	candidates = candidates.map((cand) => {	×
166	cand.answer = backend.i18n.postprocessNLG(cand.answer, entities);	×
167	return cand;	×
168	});
169
170	return candidates;	×
171	}
172
173	async function queryNLG(params, data, res) {
174	const app = res.app;	×
175
176	if (params.locale !== app.args.locale) {	×
177	res.status(400).json({ error: 'Unsupported language' });	×
178	return;	×
179	}
180
181	const result = await runNLGPrediction(app.backend, data.context, data.entities, data.target,	×
182	data.limit ? parseInt(data.limit) : 5);	×
183
184	res.json({	×
185	candidates: result,
186	});
187	}
188
189	const QUERY_PARAMS = {	52✔
190	q: 'string',
191	store: '?string',
192	access_token: '?string',
193	thingtalk_version: '?string',
194	limit: '?integer',
195	expect: '?string',
196	choices: '?array',
197	context: '?string',
198	entities: '?object',
199	tokenized: 'boolean',
200	skip_typechecking: 'boolean',
201	developer_key: '?string',
202	};
203	const NLG_PARAMS = {	52✔
204	context: 'string',
205	entities: 'object',
206	target: 'string',
207	};
208
209	module.exports = {	52✔
210	initArgparse(subparsers) {
211	const parser = subparsers.addParser('server', {	52✔
212	addHelp: true,
213	description: "Expose a Genie-compatible NLP API over HTTP."
214	});
215	parser.addArgument(['-p', '--port'], {	52✔
216	required: false,
217	help: "HTTP port to listen on",
218	defaultValue: 8400,
219	});
220	parser.addArgument('--nlu-model', {	52✔
221	required: true,
222	help: "Path to the NLU model, pointing to a model directory.",
223	});
224	parser.addArgument('--nlg-model', {	52✔
225	required: false,
226	help: "Path to the NLU model, pointing to a model directory.",
227	});
228	parser.addArgument('--thingpedia', {	52✔
229	required: true,
230	help: 'Path to ThingTalk file containing class definitions.'
231	});
232	parser.addArgument(['-l', '--locale'], {	52✔
233	required: false,
234	defaultValue: 'en-US',
235	help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
236	});
237	parser.addArgument('--debug', {	52✔
238	nargs: 0,
239	action: 'storeTrue',
240	help: 'Enable debugging.',
241	defaultValue: true
242	});
243	parser.addArgument('--no-debug', {	52✔
244	nargs: 0,
245	action: 'storeFalse',
246	dest: 'debug',
247	help: 'Disable debugging.',
248	});
249	},
250
251	async execute(args) {
252	const tpClient = new Tp.FileClient(args);	×
253	const schemas = new ThingTalk.SchemaRetriever(tpClient, null, true);	×
254	const app = express();	×
255
NEW 256	const i18n = I18n.get(args.locale);	×
UNCOV 257	app.backend = {	×
258	schemas,
259	i18n,
260	tokenizer: i18n.getTokenizer(),
261	nlu: new Predictor('nlu', args.nlu_model, 1)
262	};
263	app.backend.nlu.start();	×
264	if (args.nlg_model && args.nlg_model !== args.nlu_model) {	×
265	app.backend.nlg = new Predictor('nlg', args.nlg_model, 1);	×
266	app.backend.nlg.start();	×
267	} else {
268	app.backend.nlg = app.backend.nlu;	×
269	}
270
271	app.args = args;	×
272
273	app.set('port', args.port);	×
274	app.use(bodyParser.json());	×
275	app.use(bodyParser.urlencoded({ extended: true }));	×
276	app.use(logger('dev'));	×
277
278	app.use((req, res, next) => {	×
279	res.set('Access-Control-Allow-Origin', '*');	×
280	next();	×
281	});
282
283	app.post('/:locale/query', qv.validatePOST(QUERY_PARAMS, { accept: 'application/json', json: true }), (req, res, next) => {	×
284	queryNLU(req.params, req.body, res).catch(next);	×
285	});
286
287	app.post('/:locale/answer', qv.validatePOST(NLG_PARAMS, { accept: 'application/json', json: true }), (req, res, next) => {	×
288	queryNLG(req.params, req.body, res).catch(next);	×
289	});
290
291	app.post('/:locale/tokenize', qv.validatePOST({ q: 'string', entities: '?object' }, { accept: 'application/json', json: true }), (req, res, next) => {	×
292	tokenize(req.params, req.body, res).catch(next);	×
293	});
294
295	app.post('/:locale/learn', learn);	×
296
297	// if we get here, we have a 404 error
298	app.use('/', (req, res) => {	×
299	res.status(404).json({ error: 'Invalid endpoint' });	×
300	});
301	app.use(errorhandler());	×
302
303	const server = app.listen(app.get('port'));	×
304
305	await new Promise((resolve, reject) => {	×
306	process.on('SIGINT', resolve);	×
307	process.on('SIGTERM', resolve);	×
308	});
309
310	await app.backend.nlu.stop();	×
311	if (app.backend.nlg !== app.backend.nlu)	×
312	await app.backend.nlg.stop();	×
313	server.close();	×
314	}
315	};

stanford-oval / genie-toolkit / 885

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous