2654

pending completion

Build # 2654

Build Type

Pull #884

travis-ci-com

Committed by

web-flow

Commit Message

Merge 9242bd881 into 0846db3e0

Pull Request Pull Request #884: add support for measurement type

Run Details

10495 of 21351 branches covered (49.15%)

Branch coverage included in aggregate %.

0 of 2 new or added lines in 1 file covered. (0.0%)

5 existing lines in 3 files now uncovered.

25011 of 42045 relevant lines covered (59.49%)

306847.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.9

/tool/sample-synthetic-data.ts

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Jake Wu <jmhw0123@gmail.com>

import * as argparse from 'argparse';
import * as fs from 'fs';
import * as Tp from 'thingpedia';
import * as I18n from '../lib/i18n';
import * as utils from '../lib/utils/misc-utils';
import { Ast, Type, SchemaRetriever } from 'thingtalk';
import { ParaphraseExample, generateExamples as generateQueryExamples } from './autoqa/lib/canonical-example-constructor';
import { parseConstantFile } from './lib/constant-file';
import { getElementType } from './autoqa/wikidata/utils';
import { makeLookupKeys } from '../lib/dataset-tools/mturk/sample-utils';
import { 
    PARTS_OF_SPEECH, 
    PROJECTION_PARTS_OF_SPEECH, 
    Canonicals, 
    CanonicalAnnotation 
} from './autoqa/lib/base-canonical-generator';
import genBaseCanonical from './autoqa/lib/base-canonical-generator';
import { serializePrediction } from '../lib/utils/thingtalk';
import { EntityUtils } from '../lib';
import Path = require('path');

interface NewParaphraseExample extends ParaphraseExample {
    thingtalk : string
}

interface Constant {
    key ?: string;
    value : any;
    display : string;
    unit ?: string;
}

function typeToString(type : Type) : string {
    const elemType = getElementType(type);
    if (elemType instanceof Type.Entity)
        return elemType.type;
    return type.toString();
}

function countArgTypes(schema : Ast.FunctionDef) : Record<string, number> {
    const count : Record<string, number> = {};
    for (const arg of schema.iterateArguments()) {
        const typestr = typeToString(arg.type);
        if (!typestr)
            continue;
        count[typestr] = (count[typestr] || 0) + 1;
    }
    return count;
}

function checkOutputPath(args : any) {
    if (!args.output) {
        const outputDir = Path.join(Path.dirname(args.thingpedia), "test");
        if (!fs.existsSync(outputDir))
            fs.mkdirSync(outputDir);
        args.output = fs.createWriteStream(Path.join(outputDir, "samples.tsv"));
    }
}

function parseConstantKeys(classDef : Ast.ClassDef, 
                           sampleMeta : Record<string, Constant[]>, 
                           fname : string, 
                           arg : Ast.ArgumentDef) : Constant[] {
    //TODO: also use enum canonicals?
    const keys = makeLookupKeys('@' + classDef.kind + '.' + fname, arg.name, arg.type);
    let sampleConstants : Constant[] = [];
    for (const key of keys) {
        if (sampleMeta[key]) {
            sampleConstants = sampleMeta[key];
            break;
        }
    }
    return sampleConstants;
}

function retrieveSampleValues(classDef : Ast.ClassDef, 
                              sampleMeta : Record<string, Constant[]>, 
                              fname : string, 
                              arg : Ast.ArgumentDef) : string[] {
    if (arg.type instanceof Type.Enum) 
        return arg.type.entries!.slice(0, 10).map(utils.clean);
    const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);
    return sampleConstants.map((v) => {
        if ((arg.type === Type.String) || 
            (arg.type instanceof Type.Array && ((arg.type.elem as Type) === Type.String)))
            return v.value;
        return v.display;
    });
}

function toThingtalkValue(classDef : Ast.ClassDef, 
                          sampleMeta : Record<string, Constant[]>, 
                          fname : string, 
                          arg : Ast.ArgumentDef, 
                          value : string) : { value : Ast.Value, op : string } {
    value = value.toLowerCase();
    let type = arg.type;
    if (type instanceof Type.Entity) {
        const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);
        const kv = sampleConstants.find((item) => item.value.display?.toLowerCase() === value);
        const ttValue = kv?.value.value.toLowerCase();
        const ttDisplay = kv?.value.display.toLowerCase();
        return { value: new Ast.Value.Entity(ttValue, type.type, ttDisplay), op: "==" };
    }
    if (type instanceof Type.Enum)
        return { value: new Ast.Value.Enum(value), op: "==" };
    if (type === Type.String)
        return { value: new Ast.Value.String(value), op: "=~" };
    if (type === Type.Number)
        return { value: new Ast.Value.Number(parseFloat(value)), op: "==" };
    if (type instanceof Type.Measure)
        return { value: new Ast.Value.Measure(parseFloat(value), type.unit), op: "==" };
    if (type instanceof Type.Array) {
        type = type.elem as Type;
        if (type instanceof Type.Entity) {
            const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);
            const kv = sampleConstants.find((item) => item.value.display?.toLowerCase() === value);
            const ttValue = kv?.value.value.toLowerCase();
            const ttDisplay = kv?.value.display.toLowerCase();
            return { value: new Ast.Value.Entity(ttValue, type.type, ttDisplay), op: "contains" };
        } else if (type === Type.String) {
            return { value: new Ast.Value.String(value), op: "contains~" };
        } else {
            throw new Error(`Unsupported value type: ${type}`);
        }
    }
    throw new Error(`Unsupported value type: ${type.prettyprint()}`);
}

function toTSV(device : string, data : NewParaphraseExample[], useHeading : boolean) {
    let headings = '';
    if (useHeading)
        headings = ["id", "utterance", "thingtalk"].join('\t') + '\n';
    const rows = data.reduce((acc : string[], colValue, idx) => {
        const id = `${device}-${idx.toLocaleString('en-US', { minimumIntegerDigits: 3, useGrouping:false })}`;
        const tmp = [
            id,
            colValue["utterance"],
            colValue["thingtalk"]
        ];
        return acc.concat([tmp.join('\t')]);
    }, []).join('\n');
    return `${headings}${rows}`;
}

function generateBaseCanonicalAnnotation(func : Ast.FunctionDef, 
                                         arg : Ast.ArgumentDef, 
                                         typeCounts : Record<string, number>,
                                         queries : string[],
                                         remove_existing_canonicals : boolean) : CanonicalAnnotation {
    const canonicalAnnotation : CanonicalAnnotation = {};
    if (remove_existing_canonicals) {
        genBaseCanonical(canonicalAnnotation, arg.name, arg.type);
    } else {
        const existingCanonical : Record<string, any> = arg.getNaturalLanguageAnnotation('canonical') || {};
        if (typeof existingCanonical === 'string') 
            canonicalAnnotation.base = [existingCanonical];
        else if (Array.isArray(existingCanonical))
            canonicalAnnotation.base = existingCanonical;
        else if (typeof existingCanonical === 'object') 
            Object.assign(canonicalAnnotation, existingCanonical);
    } 

    // remove function name in arg name, normally it's repetitive
    for (const [key, value] of Object.entries(canonicalAnnotation)) {
        if (PARTS_OF_SPEECH.includes(key)) {
            canonicalAnnotation[key as keyof Canonicals] = value.map((c : string) => {
                if (c.startsWith(func.name.toLowerCase() + ' '))
                    return c.slice(func.name.toLowerCase().length + 1);
                return c;
            });
        }
    }

    // copy base canonical if property canonical is missing
    if (canonicalAnnotation.base && !canonicalAnnotation.property)
        canonicalAnnotation.property = [...canonicalAnnotation.base];

    const typestr = typeToString(func.getArgType(arg.name)!);
    if (typestr && typeCounts[typestr] === 1) {
        // if an entity is unique, allow dropping the property name entirely
        // FIXME: consider type hierarchy, or probably drop it entirely
        if (canonicalAnnotation.property && !queries.includes(typestr.substring(typestr.indexOf(':') + 1))) {
            if (!canonicalAnnotation.property.includes('#'))
                canonicalAnnotation.property.push('#');
        }

        // if property is missing, use the type information
        if (!('property' in canonicalAnnotation)) {
            const base = utils.clean(typestr.substring(typestr.indexOf(':') + 1));
            canonicalAnnotation['property'] = [base];
            canonicalAnnotation['base'] = [base];
        }

        // if it's the only people entity, adding adjective form
        // E.g., author for review - bob's review
        //       byArtist for MusicRecording - bob's song
        if (typestr.endsWith(':Person'))
            canonicalAnnotation.adjective = ["# 's", '#'];

        // if it's the only date, adding argmin/argmax/base_projection
        if (typestr === 'Date') {
            canonicalAnnotation.adjective_argmax = ["most recent", "latest", "last", "newest"];
            canonicalAnnotation.adjective_argmin = ["earliest", "first", "oldest"];
            canonicalAnnotation.base_projection = ['date'];
        }
    }
    return canonicalAnnotation;
}

export function generateActionExamples(query : Ast.FunctionDef,
                                       arg : Ast.ArgumentDef, 
                                       baseCanonicalAnnotation : CanonicalAnnotation, 
                                       sampleValues : string[]) : ParaphraseExample[] {
    const examples : ParaphraseExample[] = [];
    const queryCanonical = Array.isArray(query.nl_annotations.canonical) ? query.nl_annotations.canonical[0] : query.nl_annotations.canonical;
    for (const [pos, canonicals] of Object.entries(baseCanonicalAnnotation)) {
        if (!PARTS_OF_SPEECH.includes(pos)) 
            continue;
        for (const canonical of canonicals) {
            if (PROJECTION_PARTS_OF_SPEECH.includes(pos)) {
                examples.push(...generateActionExamplesByPOS(query, queryCanonical, arg, canonical, pos));
            } else {
                for (const value of sampleValues) 
                    examples.push(...generateActionExamplesByPOS(query, queryCanonical, arg, canonical, pos, value));
            }
        }    
    }
    return examples;
}

// FIXME: Consider renmaing ParaphraseExample's queryCanonical property so we don't cause confusion here.
function generateActionExamplesByPOS(action : Ast.FunctionDef,
                                     queryCanonical : string,
                                     argument : Ast.ArgumentDef,
                                     argumentCanonical : string,
                                     pos : string, 
                                     value ?: string|boolean) : ParaphraseExample[] {
    function example(utterance : string) : ParaphraseExample {
        return { query: action.name, queryCanonical, argument: argument.name, utterance, value, paraphrases : [] };
    }
    if (!PROJECTION_PARTS_OF_SPEECH.includes(pos)) {
        if (!argumentCanonical.includes('#'))
            argumentCanonical = argumentCanonical + ' #';
    }
    const predicate = typeof value === 'string' ? argumentCanonical.replace('#', value) : argumentCanonical;
    switch (pos) {
    case 'base':
        return [
            example(`${queryCanonical}`),
            example(`${queryCanonical} with the ${argumentCanonical}?`),
        ];
    case 'property':
    case 'property_true':
    case 'property_false':
        return [
            example(`${queryCanonical} with ${predicate}.`),
        ];
    case 'verb':
    case 'verb_true':
    case 'verb_false':
        return [
            example(`${queryCanonical} that ${predicate}.`),
        ];
    case 'passive_verb':
    case 'passive_verb_true':
    case 'passive_verb_false':
    case 'preposition':
    case 'preposition_true':
    case 'preposition_false':
        return [
            example(`${queryCanonical} ${predicate}.`),
        ];
    case 'reverse_property':
    case 'reverse_property_true':
    case 'reverse_property_false':
        return [
            example(`${queryCanonical} ${predicate}?`)
        ];
    case 'adjective':
    case 'adjective_true':
    case 'adjective_false':
        return [
            example(`${queryCanonical} ${predicate} .`),
        ];
    case 'reverse_verb':
        return [
            example(`${queryCanonical} ${predicate}`)
        ];
    default:
        return [];
    }
    
}

function generateFilterAst(device : string, 
                           func : string, 
                           property : string, 
                           operator : string, 
                           value : any) : Ast.Program {
    const invocation = new Ast.InvocationExpression(
        null, 
        new Ast.Invocation(null, new Ast.DeviceSelector(null, device, null, null), func, [], null),
        null
    );
    const filter = new Ast.AtomBooleanExpression(
        null,
        property, 
        operator, 
        value, // 
        null
    );
    const filtered = new Ast.FilterExpression(
        null, 
        invocation, 
        filter, 
        null
    );
    const statement = new Ast.ExpressionStatement(
        null,
        new Ast.ChainExpression(null, [filtered], null)
    );
    return new Ast.Program(
        null,
        [], 
        [],
        [statement],
        {}
    );
}

function generateProjectionAst(device : string, func : string, property : string) : Ast.Program {
    const invocation = new Ast.InvocationExpression(
        null, 
        new Ast.Invocation(null, new Ast.DeviceSelector(null, device, null, null), func, [], null),
        null
    );
    const projection = new Ast.ProjectionExpression(
        null,
        invocation, 
        [property],
        [], 
        [], 
        null
    );
    const statement = new Ast.ExpressionStatement(
        null,
        new Ast.ChainExpression(null, [projection], null)
    );
    return new Ast.Program(
        null,
        [], 
        [],
        [statement],
        {}
    );
}

function generateActionAst(func : string, property : string, value : any) : Ast.Program {
    const inputParam = new Ast.InputParam(null, property, value);
    const action = new Ast.FunctionCallExpression(
        null,
        func, 
        [inputParam],
        null
    );
    const statement = new Ast.ExpressionStatement(
        null,
        new Ast.ChainExpression(null, [action], null)
    );
    return new Ast.Program(
        null,
        [], 
        [],
        [statement],
        {}
    );
}

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('sample-synthetic-data', {
        add_help: true,
        description: "Automatically generate samples from the canonicals"
    });
    parser.add_argument('-o', '--output', {
        required: false,
        type: fs.createWriteStream
    });
    parser.add_argument('-l', '--locale', {
        default: 'en-US',
        help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
    });
    parser.add_argument('-c', '--constants', {
        required: false,
        help: 'TSV file containing sampled constant values to be used.'
    });
    parser.add_argument('-t', '--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('-d', '--device', {
        required: true,
        help: `The name of the device to be synthesized.`
    });
    parser.add_argument('-f', '--function', {
        required: false,
        help: `A specific function to be sampled`
    });
}

export async function execute(args : any) {
    process.stdout.write("Generating samples... ");
    checkOutputPath(args);
    const tpClient = new Tp.FileClient(args);
    const schemaRetriever = new SchemaRetriever(tpClient, null, false);
    const deviceClass = await schemaRetriever.getFullSchema(args.device);
    const baseTokenizer : I18n.BaseTokenizer = I18n.get(args.locale).getTokenizer();
    const functionNames = Object.keys(deviceClass.queries).concat(Object.keys(deviceClass.actions));
    const sampleMeta = await parseConstantFile(args.locale, args.constants);
    const utteranceThingtalkPairs : NewParaphraseExample[] = [];
    const options = { locale: args.locale, timezone: undefined, includeEntityValue: true };
    for (const fname of functionNames) {
        if (args.function && fname !== args.function)
            continue;
        const func = deviceClass.queries[fname] || deviceClass.actions[fname];
        const typeCounts = countArgTypes(func);
        for (const arg of func.iterateArguments()) {
            const sampleValues = retrieveSampleValues(deviceClass, sampleMeta, fname, arg);
            const canonicalAnnotation = generateBaseCanonicalAnnotation(func, arg, typeCounts, functionNames, false);
            let utteranceExamples : ParaphraseExample[];
            if (deviceClass.actions[fname])
                utteranceExamples = generateActionExamples(func, arg, canonicalAnnotation, sampleValues);
            else
                utteranceExamples = generateQueryExamples(func, arg, canonicalAnnotation, sampleValues);
            for (const ex of utteranceExamples) {
                const example = ex as NewParaphraseExample;
                const prepUtterance = baseTokenizer.tokenize(example.utterance).tokens.join(' ');
                let program : Ast.Program;
                if (deviceClass.actions[fname]) {
                    if (example.value) {
                        const { value, } = toThingtalkValue(deviceClass, sampleMeta, fname, arg, `${example.value}`);
                        program = generateActionAst(fname, example.argument, value);
                    } else {
                        continue;
                    }
                } else {
                    if (example.value) {
                        const { value, op } = toThingtalkValue(deviceClass, sampleMeta, fname, arg, `${example.value}`);
                        program = generateFilterAst(args.device, fname, example.argument, op, value);
                    } else {
                        program = generateProjectionAst(args.device, fname, example.argument);
                    }
                }
                try {
                    const entityDummy = EntityUtils.makeDummyEntities(prepUtterance);
                    example.thingtalk = serializePrediction(program, prepUtterance, entityDummy, options).join(' ');
                } catch(err) {
                    console.log(prepUtterance);
                    console.log(program.prettyprint());
                    throw err;
                }
                example.utterance = prepUtterance;
                utteranceThingtalkPairs.push(example);
            }
        }
    }
    const output = toTSV(args.device, utteranceThingtalkPairs, false);
    // console.log(output);
    args.output.write(output);
    process.stdout.write(`Done!\nFile location: ${args.output.path}\n`);
}

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Jake Wu <jmhw0123@gmail.com>
20
21	import * as argparse from 'argparse';
22	import * as fs from 'fs';	96✔
23	import * as Tp from 'thingpedia';	96✔
24	import * as I18n from '../lib/i18n';	96✔
25	import * as utils from '../lib/utils/misc-utils';	96✔
26	import { Ast, Type, SchemaRetriever } from 'thingtalk';	96✔
27	import { ParaphraseExample, generateExamples as generateQueryExamples } from './autoqa/lib/canonical-example-constructor';	96✔
28	import { parseConstantFile } from './lib/constant-file';	96✔
29	import { getElementType } from './autoqa/wikidata/utils';	96✔
30	import { makeLookupKeys } from '../lib/dataset-tools/mturk/sample-utils';	96✔
31	import {	96✔
32	PARTS_OF_SPEECH,
33	PROJECTION_PARTS_OF_SPEECH,
34	Canonicals,
35	CanonicalAnnotation
36	} from './autoqa/lib/base-canonical-generator';
37	import genBaseCanonical from './autoqa/lib/base-canonical-generator';	96✔
38	import { serializePrediction } from '../lib/utils/thingtalk';	96✔
39	import { EntityUtils } from '../lib';	96✔
40	import Path = require('path');	96✔
41
42	interface NewParaphraseExample extends ParaphraseExample {
43	thingtalk : string
44	}
45
46	interface Constant {
47	key ?: string;
48	value : any;
49	display : string;
50	unit ?: string;
51	}
52
53	function typeToString(type : Type) : string {
54	const elemType = getElementType(type);	×
55	if (elemType instanceof Type.Entity)	×
56	return elemType.type;	×
57	return type.toString();	×
58	}
59
60	function countArgTypes(schema : Ast.FunctionDef) : Record<string, number> {
61	const count : Record<string, number> = {};	×
62	for (const arg of schema.iterateArguments()) {	×
63	const typestr = typeToString(arg.type);	×
64	if (!typestr)	×
65	continue;	×
66	count[typestr] = (count[typestr] \|\| 0) + 1;	×
67	}
68	return count;	×
69	}
70
71	function checkOutputPath(args : any) {
72	if (!args.output) {	×
73	const outputDir = Path.join(Path.dirname(args.thingpedia), "test");	×
74	if (!fs.existsSync(outputDir))	×
75	fs.mkdirSync(outputDir);	×
76	args.output = fs.createWriteStream(Path.join(outputDir, "samples.tsv"));	×
77	}
78	}
79
80	function parseConstantKeys(classDef : Ast.ClassDef,
81	sampleMeta : Record<string, Constant[]>,
82	fname : string,
83	arg : Ast.ArgumentDef) : Constant[] {
84	//TODO: also use enum canonicals?
85	const keys = makeLookupKeys('@' + classDef.kind + '.' + fname, arg.name, arg.type);	×
86	let sampleConstants : Constant[] = [];	×
87	for (const key of keys) {	×
88	if (sampleMeta[key]) {	×
89	sampleConstants = sampleMeta[key];	×
90	break;	×
91	}
92	}
93	return sampleConstants;	×
94	}
95
96	function retrieveSampleValues(classDef : Ast.ClassDef,
97	sampleMeta : Record<string, Constant[]>,
98	fname : string,
99	arg : Ast.ArgumentDef) : string[] {
100	if (arg.type instanceof Type.Enum)	×
101	return arg.type.entries!.slice(0, 10).map(utils.clean);	×
102	const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);	×
103	return sampleConstants.map((v) => {	×
104	if ((arg.type === Type.String) \|\|	×
105	(arg.type instanceof Type.Array && ((arg.type.elem as Type) === Type.String)))
106	return v.value;	×
107	return v.display;	×
108	});
109	}
110
111	function toThingtalkValue(classDef : Ast.ClassDef,
112	sampleMeta : Record<string, Constant[]>,
113	fname : string,
114	arg : Ast.ArgumentDef,
115	value : string) : { value : Ast.Value, op : string } {
116	value = value.toLowerCase();	×
117	let type = arg.type;	×
118	if (type instanceof Type.Entity) {	×
119	const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);	×
120	const kv = sampleConstants.find((item) => item.value.display?.toLowerCase() === value);	×
121	const ttValue = kv?.value.value.toLowerCase();	×
122	const ttDisplay = kv?.value.display.toLowerCase();	×
123	return { value: new Ast.Value.Entity(ttValue, type.type, ttDisplay), op: "==" };	×
124	}
125	if (type instanceof Type.Enum)	×
126	return { value: new Ast.Value.Enum(value), op: "==" };	×
127	if (type === Type.String)	×
128	return { value: new Ast.Value.String(value), op: "=~" };	×
129	if (type === Type.Number)	×
130	return { value: new Ast.Value.Number(parseFloat(value)), op: "==" };	×
NEW 131	if (type instanceof Type.Measure)	×
NEW 132	return { value: new Ast.Value.Measure(parseFloat(value), type.unit), op: "==" };	×
133	if (type instanceof Type.Array) {	×
134	type = type.elem as Type;	×
135	if (type instanceof Type.Entity) {	×
136	const sampleConstants = parseConstantKeys(classDef, sampleMeta, fname, arg);	×
137	const kv = sampleConstants.find((item) => item.value.display?.toLowerCase() === value);	×
138	const ttValue = kv?.value.value.toLowerCase();	×
139	const ttDisplay = kv?.value.display.toLowerCase();	×
140	return { value: new Ast.Value.Entity(ttValue, type.type, ttDisplay), op: "contains" };	×
141	} else if (type === Type.String) {	×
142	return { value: new Ast.Value.String(value), op: "contains~" };	×
143	} else {
144	throw new Error(`Unsupported value type: ${type}`);	×
145	}
146	}
147	throw new Error(`Unsupported value type: ${type.prettyprint()}`);	×
148	}
149
150	function toTSV(device : string, data : NewParaphraseExample[], useHeading : boolean) {
151	let headings = '';	×
152	if (useHeading)	×
153	headings = ["id", "utterance", "thingtalk"].join('\t') + '\n';	×
154	const rows = data.reduce((acc : string[], colValue, idx) => {	×
155	const id = `${device}-${idx.toLocaleString('en-US', { minimumIntegerDigits: 3, useGrouping:false })}`;	×
156	const tmp = [	×
157	id,
158	colValue["utterance"],
159	colValue["thingtalk"]
160	];
161	return acc.concat([tmp.join('\t')]);	×
162	}, []).join('\n');
163	return `${headings}${rows}`;	×
164	}
165
166	function generateBaseCanonicalAnnotation(func : Ast.FunctionDef,
167	arg : Ast.ArgumentDef,
168	typeCounts : Record<string, number>,
169	queries : string[],
170	remove_existing_canonicals : boolean) : CanonicalAnnotation {
171	const canonicalAnnotation : CanonicalAnnotation = {};	×
172	if (remove_existing_canonicals) {	×
173	genBaseCanonical(canonicalAnnotation, arg.name, arg.type);	×
174	} else {
175	const existingCanonical : Record<string, any> = arg.getNaturalLanguageAnnotation('canonical') \|\| {};	×
176	if (typeof existingCanonical === 'string')	×
177	canonicalAnnotation.base = [existingCanonical];	×
178	else if (Array.isArray(existingCanonical))	×
179	canonicalAnnotation.base = existingCanonical;	×
180	else if (typeof existingCanonical === 'object')	×
181	Object.assign(canonicalAnnotation, existingCanonical);	×
182	}
183
184	// remove function name in arg name, normally it's repetitive
185	for (const [key, value] of Object.entries(canonicalAnnotation)) {	×
186	if (PARTS_OF_SPEECH.includes(key)) {	×
187	canonicalAnnotation[key as keyof Canonicals] = value.map((c : string) => {	×
188	if (c.startsWith(func.name.toLowerCase() + ' '))	×
189	return c.slice(func.name.toLowerCase().length + 1);	×
190	return c;	×
191	});
192	}
193	}
194
195	// copy base canonical if property canonical is missing
196	if (canonicalAnnotation.base && !canonicalAnnotation.property)	×
197	canonicalAnnotation.property = [...canonicalAnnotation.base];	×
198
199	const typestr = typeToString(func.getArgType(arg.name)!);	×
200	if (typestr && typeCounts[typestr] === 1) {	×
201	// if an entity is unique, allow dropping the property name entirely
202	// FIXME: consider type hierarchy, or probably drop it entirely
203	if (canonicalAnnotation.property && !queries.includes(typestr.substring(typestr.indexOf(':') + 1))) {	×
204	if (!canonicalAnnotation.property.includes('#'))	×
205	canonicalAnnotation.property.push('#');	×
206	}
207
208	// if property is missing, use the type information
209	if (!('property' in canonicalAnnotation)) {	×
210	const base = utils.clean(typestr.substring(typestr.indexOf(':') + 1));	×
211	canonicalAnnotation['property'] = [base];	×
212	canonicalAnnotation['base'] = [base];	×
213	}
214
215	// if it's the only people entity, adding adjective form
216	// E.g., author for review - bob's review
217	// byArtist for MusicRecording - bob's song
218	if (typestr.endsWith(':Person'))	×
219	canonicalAnnotation.adjective = ["# 's", '#'];	×
220
221	// if it's the only date, adding argmin/argmax/base_projection
222	if (typestr === 'Date') {	×
223	canonicalAnnotation.adjective_argmax = ["most recent", "latest", "last", "newest"];	×
224	canonicalAnnotation.adjective_argmin = ["earliest", "first", "oldest"];	×
225	canonicalAnnotation.base_projection = ['date'];	×
226	}
227	}
228	return canonicalAnnotation;	×
229	}
230
231	export function generateActionExamples(query : Ast.FunctionDef,	96✔
232	arg : Ast.ArgumentDef,
233	baseCanonicalAnnotation : CanonicalAnnotation,
234	sampleValues : string[]) : ParaphraseExample[] {
235	const examples : ParaphraseExample[] = [];	×
236	const queryCanonical = Array.isArray(query.nl_annotations.canonical) ? query.nl_annotations.canonical[0] : query.nl_annotations.canonical;	×
237	for (const [pos, canonicals] of Object.entries(baseCanonicalAnnotation)) {	×
238	if (!PARTS_OF_SPEECH.includes(pos))	×
239	continue;	×
240	for (const canonical of canonicals) {	×
241	if (PROJECTION_PARTS_OF_SPEECH.includes(pos)) {	×
242	examples.push(...generateActionExamplesByPOS(query, queryCanonical, arg, canonical, pos));	×
243	} else {
244	for (const value of sampleValues)	×
245	examples.push(...generateActionExamplesByPOS(query, queryCanonical, arg, canonical, pos, value));	×
246	}
247	}
248	}
249	return examples;	×
250	}
251
252	// FIXME: Consider renmaing ParaphraseExample's queryCanonical property so we don't cause confusion here.
253	function generateActionExamplesByPOS(action : Ast.FunctionDef,
254	queryCanonical : string,
255	argument : Ast.ArgumentDef,
256	argumentCanonical : string,
257	pos : string,
258	value ?: string\|boolean) : ParaphraseExample[] {
259	function example(utterance : string) : ParaphraseExample {
260	return { query: action.name, queryCanonical, argument: argument.name, utterance, value, paraphrases : [] };	×
261	}
262	if (!PROJECTION_PARTS_OF_SPEECH.includes(pos)) {	×
263	if (!argumentCanonical.includes('#'))	×
264	argumentCanonical = argumentCanonical + ' #';	×
265	}
266	const predicate = typeof value === 'string' ? argumentCanonical.replace('#', value) : argumentCanonical;	×
267	switch (pos) {	×
268	case 'base':
269	return [	×
270	example(`${queryCanonical}`),
271	example(`${queryCanonical} with the ${argumentCanonical}?`),
272	];
273	case 'property':
274	case 'property_true':
275	case 'property_false':
276	return [	×
277	example(`${queryCanonical} with ${predicate}.`),
278	];
279	case 'verb':
280	case 'verb_true':
281	case 'verb_false':
282	return [	×
283	example(`${queryCanonical} that ${predicate}.`),
284	];
285	case 'passive_verb':
286	case 'passive_verb_true':
287	case 'passive_verb_false':
288	case 'preposition':
289	case 'preposition_true':
290	case 'preposition_false':
291	return [	×
292	example(`${queryCanonical} ${predicate}.`),
293	];
294	case 'reverse_property':
295	case 'reverse_property_true':
296	case 'reverse_property_false':
297	return [	×
298	example(`${queryCanonical} ${predicate}?`)
299	];
300	case 'adjective':
301	case 'adjective_true':
302	case 'adjective_false':
303	return [	×
304	example(`${queryCanonical} ${predicate} .`),
305	];
306	case 'reverse_verb':
307	return [	×
308	example(`${queryCanonical} ${predicate}`)
309	];
310	default:
311	return [];	×
312	}
313
314	}
315
316	function generateFilterAst(device : string,
317	func : string,
318	property : string,
319	operator : string,
320	value : any) : Ast.Program {
321	const invocation = new Ast.InvocationExpression(	×
322	null,
323	new Ast.Invocation(null, new Ast.DeviceSelector(null, device, null, null), func, [], null),
324	null
325	);
326	const filter = new Ast.AtomBooleanExpression(	×
327	null,
328	property,
329	operator,
330	value, //
331	null
332	);
333	const filtered = new Ast.FilterExpression(	×
334	null,
335	invocation,
336	filter,
337	null
338	);
339	const statement = new Ast.ExpressionStatement(	×
340	null,
341	new Ast.ChainExpression(null, [filtered], null)
342	);
343	return new Ast.Program(	×
344	null,
345	[],
346	[],
347	[statement],
348	{}
349	);
350	}
351
352	function generateProjectionAst(device : string, func : string, property : string) : Ast.Program {
353	const invocation = new Ast.InvocationExpression(	×
354	null,
355	new Ast.Invocation(null, new Ast.DeviceSelector(null, device, null, null), func, [], null),
356	null
357	);
358	const projection = new Ast.ProjectionExpression(	×
359	null,
360	invocation,
361	[property],
362	[],
363	[],
364	null
365	);
366	const statement = new Ast.ExpressionStatement(	×
367	null,
368	new Ast.ChainExpression(null, [projection], null)
369	);
370	return new Ast.Program(	×
371	null,
372	[],
373	[],
374	[statement],
375	{}
376	);
377	}
378
379	function generateActionAst(func : string, property : string, value : any) : Ast.Program {
380	const inputParam = new Ast.InputParam(null, property, value);	×
381	const action = new Ast.FunctionCallExpression(	×
382	null,
383	func,
384	[inputParam],
385	null
386	);
387	const statement = new Ast.ExpressionStatement(	×
388	null,
389	new Ast.ChainExpression(null, [action], null)
390	);
391	return new Ast.Program(	×
392	null,
393	[],
394	[],
395	[statement],
396	{}
397	);
398	}
399
400	export function initArgparse(subparsers : argparse.SubParser) {	96✔
401	const parser = subparsers.add_parser('sample-synthetic-data', {	96✔
402	add_help: true,
403	description: "Automatically generate samples from the canonicals"
404	});
405	parser.add_argument('-o', '--output', {	96✔
406	required: false,
407	type: fs.createWriteStream
408	});
409	parser.add_argument('-l', '--locale', {	96✔
410	default: 'en-US',
411	help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
412	});
413	parser.add_argument('-c', '--constants', {	96✔
414	required: false,
415	help: 'TSV file containing sampled constant values to be used.'
416	});
417	parser.add_argument('-t', '--thingpedia', {	96✔
418	required: true,
419	help: 'Path to ThingTalk file containing class definitions.'
420	});
421	parser.add_argument('-d', '--device', {	96✔
422	required: true,
423	help: `The name of the device to be synthesized.`
424	});
425	parser.add_argument('-f', '--function', {	96✔
426	required: false,
427	help: `A specific function to be sampled`
428	});
429	}
430
431	export async function execute(args : any) {	96✔
432	process.stdout.write("Generating samples... ");	×
433	checkOutputPath(args);	×
434	const tpClient = new Tp.FileClient(args);	×
435	const schemaRetriever = new SchemaRetriever(tpClient, null, false);	×
436	const deviceClass = await schemaRetriever.getFullSchema(args.device);	×
437	const baseTokenizer : I18n.BaseTokenizer = I18n.get(args.locale).getTokenizer();	×
438	const functionNames = Object.keys(deviceClass.queries).concat(Object.keys(deviceClass.actions));	×
439	const sampleMeta = await parseConstantFile(args.locale, args.constants);	×
440	const utteranceThingtalkPairs : NewParaphraseExample[] = [];	×
441	const options = { locale: args.locale, timezone: undefined, includeEntityValue: true };	×
442	for (const fname of functionNames) {	×
443	if (args.function && fname !== args.function)	×
444	continue;	×
445	const func = deviceClass.queries[fname] \|\| deviceClass.actions[fname];	×
446	const typeCounts = countArgTypes(func);	×
447	for (const arg of func.iterateArguments()) {	×
448	const sampleValues = retrieveSampleValues(deviceClass, sampleMeta, fname, arg);	×
449	const canonicalAnnotation = generateBaseCanonicalAnnotation(func, arg, typeCounts, functionNames, false);	×
450	let utteranceExamples : ParaphraseExample[];
451	if (deviceClass.actions[fname])	×
452	utteranceExamples = generateActionExamples(func, arg, canonicalAnnotation, sampleValues);	×
453	else
454	utteranceExamples = generateQueryExamples(func, arg, canonicalAnnotation, sampleValues);	×
455	for (const ex of utteranceExamples) {	×
456	const example = ex as NewParaphraseExample;	×
457	const prepUtterance = baseTokenizer.tokenize(example.utterance).tokens.join(' ');	×
458	let program : Ast.Program;
459	if (deviceClass.actions[fname]) {	×
460	if (example.value) {	×
461	const { value, } = toThingtalkValue(deviceClass, sampleMeta, fname, arg, `${example.value}`);	×
462	program = generateActionAst(fname, example.argument, value);	×
463	} else {
464	continue;	×
465	}
466	} else {
467	if (example.value) {	×
468	const { value, op } = toThingtalkValue(deviceClass, sampleMeta, fname, arg, `${example.value}`);	×
469	program = generateFilterAst(args.device, fname, example.argument, op, value);	×
470	} else {
471	program = generateProjectionAst(args.device, fname, example.argument);	×
472	}
473	}
474	try {	×
475	const entityDummy = EntityUtils.makeDummyEntities(prepUtterance);	×
476	example.thingtalk = serializePrediction(program, prepUtterance, entityDummy, options).join(' ');	×
477	} catch(err) {
478	console.log(prepUtterance);	×
479	console.log(program.prettyprint());	×
480	throw err;	×
481	}
482	example.utterance = prepUtterance;	×
483	utteranceThingtalkPairs.push(example);	×
484	}
485	}
486	}
487	const output = toTSV(args.device, utteranceThingtalkPairs, false);	×
488	// console.log(output);
489	args.output.write(output);	×
490	process.stdout.write(`Done!\nFile location: ${args.output.path}\n`);	×
491	}

stanford-oval / genie-toolkit / 2654

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous