1340

pending completion

Build # 1340

Build Type

Pull #361

travis-ci-com

Committed by

web-flow

Commit Message

Merge 2e9e413ec into c36935d07

Pull Request Pull Request #361: Add "new" in front of all Type constructors

Run Details

6919 of 13031 branches covered (53.1%)

Branch coverage included in aggregate %.

17 of 34 new or added lines in 10 files covered. (50.0%)

3 existing lines in 3 files now uncovered.

15596 of 24860 relevant lines covered (62.74%)

1646024.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.79

/tool/sample.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
"use strict";

const seedrandom = require('seedrandom');
const fs = require('fs');
const byline = require('byline');
const csvstringify = require('csv-stringify');
const Stream = require('stream');
const Tp = require('thingpedia');
const ThingTalk = require('thingtalk');

const { DatasetParser } = require('../lib/dataset-tools/parsers');
const SentenceSampler = require('../lib/dataset-tools/mturk/sampler');
const StreamUtils = require('../lib/utils/stream-utils');
const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');
const { parseConstantFile } = require('./lib/constant-file');

function parseSamplingControlFile(filename) {
    const functionBlackList = new Set;
    const deviceBlackList = new Set;
    const functionHighValueList = new Set;
    let functionWhiteList;
    let deviceWhiteList;

    if (!filename)
        return Promise.resolve([functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList]);

    const file = fs.createReadStream(filename);
    file.setEncoding('utf8');
    const input = byline(file);


    input.on('data', (line) => {
        if (/^\s*(#|$)/.test(line))
            return;

        const [attribute, functionName] = line.trim().split('\t');

        switch (attribute) {
        case 'whitelist':
            if (functionName.endsWith('.*')) {
                if (!deviceWhiteList)
                    deviceWhiteList = new Set;
                deviceWhiteList.add(functionName);
            } else {
                if (!functionWhiteList)
                    functionWhiteList = new Set;
                functionWhiteList.add(functionName);
            }
            break;
        case 'blacklist':
            if (functionName.endsWith('.*'))
                deviceBlackList.add(functionName);
            else
                functionBlackList.add(functionName);
            break;
        case 'high':
            // ignore high value whole devices
            if (!functionName.endsWith('.*'))
                functionHighValueList.add(functionName);
            break;
        case 'low':
            // ignore low value entry (everything is low-value by default)
            break;
        default:
            throw new Error(`Invalid function attribute ${attribute}`);
        }
    });

    return new Promise((resolve, reject) => {
        input.on('end', () => resolve([functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList]));
        input.on('error', reject);
    });
}

const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;

class ContextSourceLoader extends Stream.Writable {
    constructor() {
        super({ objectMode: true });
        
        this._buffer = new Map;
    }
    
    // renumber entities so they are assigned in program order rather than sentence order
    _normalizeEntities(ex) {
        let renameMap = Object.create(null);
        let countMap = Object.create(null);
        
        let inString = false;
        let anyRename = false;
        const newCode = [];
        const code = ex.target_code.split(' ');
        for (let i = 0; i < code.length; i++) {
            const token = code[i];
            if (token === '"') {
                inString = !inString;
                newCode.push(token);
                continue;
            }
            if (inString) {
                newCode.push(token);
                continue;
            }
            if (token in renameMap) {
                newCode.push(renameMap[token]);
                
                // if we renamed a number into a measure, skip the unit
                if (token.startsWith('NUMBER_') && renameMap[token].startsWith('MEASURE_'))
                    i++;
            }
            
            const match = ENTITY_MATCH_REGEX.exec(token);
            if (match === null) {
                newCode.push(token);
                continue;
            }
            
            let [,type,] = match;
            
            // collapse NUMBER_ followed by unit into measure or duration
            if (type === 'NUMBER' && i < code.length-1 && code[i+1].startsWith('unit:')) {
                const unit = code[i+1].substring('unit:'.length);
                const baseunit = new ThingTalk.Type.Measure(unit).unit;
                if (baseunit === 'ms')
                    type = 'DURATION';
                else
                    type = 'MEASURE_' + baseunit;
                // skip the unit
                i++;
            }
            
            let newIdx;
            if (type in countMap)
                newIdx = countMap[type] + 1;
            else
                newIdx = 0;
            countMap[type] = newIdx;
            
            const newToken = type + '_' + newIdx;
            if (newToken !== token)
                anyRename = true;
            renameMap[token] = newToken;
            newCode.push(newToken);
        }
        ex.target_code = newCode.join(' ');
        if (!anyRename)
            return;
        ex.preprocessed = ex.preprocessed.split(' ').map((token) => {
            if (token in renameMap)
                return renameMap[token];
            else
                return token;
        }).join(' ');
    }
    
    _write(ex, encoding, callback) {
        /*let yes = false;
        if (ex.target_code === 'now => @com.foradb.findBP param:edate:Date = end_of unit:week - DURATION_1 param:patient:String = QUOTED_STRING_0 param:sdate:Date = end_of unit:mon + DURATION_0 => notify')
            yes = true;*/
        this._normalizeEntities(ex);
        //if (yes)
        //    console.log(ex);
    
        if (this._buffer.has(ex.target_code))
            this._buffer.get(ex.target_code).push(ex.preprocessed);
        else
            this._buffer.set(ex.target_code, [ex.preprocessed]);
        callback();
    }
    
    _finish(callback) {
        callback();
    }
    
    async read() {
        await StreamUtils.waitFinish(this);
        return this._buffer;
    }
}

module.exports = {
    initArgparse(subparsers) {
        const parser = subparsers.add_parser('sample', {
            add_help: true,
            description: "Choose which sentences to paraphrase, given a synthetic set."
        });
        parser.add_argument('-o', '--output', {
            required: true,
            type: fs.createWriteStream
        });
        parser.add_argument('--thingpedia', {
            required: true,
            help: 'Path to JSON file containing signature, type and mixin definitions.'
        });
        parser.add_argument('--constants', {
            required: true,
            help: 'TSV file containing constant values to use.'
        });
        parser.add_argument('input_file', {
            nargs: '+',
            type: maybeCreateReadStream,
            help: 'Input datasets to augment (in TSV format); use - for standard input'
        });
        parser.add_argument('-l', '--locale', {
            default: 'en-US',
            help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
        });
        parser.add_argument('--contextual', {
            action: 'store_true',
            help: 'Process a contextual dataset.',
            default: false
        });
        parser.add_argument('--context-source', {
            help: 'Source dataset from where contexts were extracted; used to choose a context sentence for each context.',
        });
        parser.add_argument('--sampling-strategy', {
            required: false,
            choices: ['byCode', 'bySentence', 'bySignature'],
            help: 'Which sampling strategy to use (defaults: bySignature).'
        });
        parser.add_argument('--sampling-control', {
            required: false,
            help: 'TSV file controlling sampling based on functions in the programs. Defaults to treating all functions equally.'
        });
        parser.add_argument('--compound-only', {
            help: 'Keep only compound programs. (False if omitted)',
            action: 'store_true'
        });

        parser.add_argument('--debug', {
            action: 'store_true',
            help: 'Enable debugging.',
            default: true
        });
        parser.add_argument('--no-debug', {
            action: 'store_false',
            dest: 'debug',
            help: 'Disable debugging.',
        });
        parser.add_argument('--random-seed', {
            default: 'almond is awesome',
            help: 'Random seed'
        });
    },

    async execute(args) {
        let contexts;
        if (args.contextual) {
            if (!args.context_source)
                throw new Error(`--context-source is required if --contextual`);

            contexts = await readAllLines([fs.createReadStream(args.context_source)])
                .pipe(new DatasetParser({ preserveId: true }))
                .pipe(new ContextSourceLoader())
                .read();
        }

        const constants = await parseConstantFile(args.locale, args.constants);
        const [functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList] =
            await parseSamplingControlFile(args.sampling_control);

        const tpClient = new Tp.FileClient(args.locale, args.thingpedia, null);
        const schemaRetriever = new ThingTalk.SchemaRetriever(tpClient, null, !args.debug);

        const options = {
            rng: seedrandom.alea(args.random_seed),
            locale: args.locale,

            samplingStrategy: args.sampling_strategy,
            functionBlackList,
            deviceBlackList,
            functionHighValueList,
            functionWhiteList,
            deviceWhiteList,
            contexts,

            compoundOnly: !!args.compound_only,
            debug: args.debug
        };

        readAllLines(args.input_file)
            .pipe(new DatasetParser({ contextual: args.contextual, preserveId: true }))
            .pipe(new SentenceSampler(schemaRetriever, constants, options))
            .pipe(csvstringify({ header: true, delimiter: '\t' }))
            .pipe(args.output);

        return new Promise((resolve, reject) => {
            args.output.on('finish', resolve);
            args.output.on('error', reject);
        });
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20	"use strict";
21
22	const seedrandom = require('seedrandom');	51✔
23	const fs = require('fs');	51✔
24	const byline = require('byline');	51✔
25	const csvstringify = require('csv-stringify');	51✔
26	const Stream = require('stream');	51✔
27	const Tp = require('thingpedia');	51✔
28	const ThingTalk = require('thingtalk');	51✔
29
30	const { DatasetParser } = require('../lib/dataset-tools/parsers');	51✔
31	const SentenceSampler = require('../lib/dataset-tools/mturk/sampler');	51✔
32	const StreamUtils = require('../lib/utils/stream-utils');	51✔
33	const { maybeCreateReadStream, readAllLines } = require('./lib/argutils');	51✔
34	const { parseConstantFile } = require('./lib/constant-file');	51✔
35
36	function parseSamplingControlFile(filename) {
37	const functionBlackList = new Set;	1✔
38	const deviceBlackList = new Set;	1✔
39	const functionHighValueList = new Set;	1✔
40	let functionWhiteList;
41	let deviceWhiteList;
42
43	if (!filename)	1!
44	return Promise.resolve([functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList]);	×
45
46	const file = fs.createReadStream(filename);	1✔
47	file.setEncoding('utf8');	1✔
48	const input = byline(file);	1✔
49
50
51	input.on('data', (line) => {	1✔
52	if (/^\s*(#\|$)/.test(line))	18!
53	return;	×
54
55	const [attribute, functionName] = line.trim().split('\t');	18✔
56
57	switch (attribute) {	18!
58	case 'whitelist':
59	if (functionName.endsWith('.*')) {	×
60	if (!deviceWhiteList)	×
61	deviceWhiteList = new Set;	×
62	deviceWhiteList.add(functionName);	×
63	} else {
64	if (!functionWhiteList)	×
65	functionWhiteList = new Set;	×
66	functionWhiteList.add(functionName);	×
67	}
68	break;	×
69	case 'blacklist':
70	if (functionName.endsWith('.*'))	×
71	deviceBlackList.add(functionName);	×
72	else
73	functionBlackList.add(functionName);	×
74	break;	×
75	case 'high':
76	// ignore high value whole devices
77	if (!functionName.endsWith('.*'))	18!
78	functionHighValueList.add(functionName);	18✔
79	break;	18✔
80	case 'low':
81	// ignore low value entry (everything is low-value by default)
82	break;	×
83	default:
84	throw new Error(`Invalid function attribute ${attribute}`);	×
85	}
86	});
87
88	return new Promise((resolve, reject) => {	1✔
89	input.on('end', () => resolve([functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList]));	1✔
90	input.on('error', reject);	1✔
91	});
92	}
93
94	const ENTITY_MATCH_REGEX = /^([A-Z].*)_[0-9]+$/;	51✔
95
96	class ContextSourceLoader extends Stream.Writable {
97	constructor() {
98	super({ objectMode: true });	×
99
100	this._buffer = new Map;	×
101	}
102
103	// renumber entities so they are assigned in program order rather than sentence order
104	_normalizeEntities(ex) {
105	let renameMap = Object.create(null);	×
106	let countMap = Object.create(null);	×
107
108	let inString = false;	×
109	let anyRename = false;	×
110	const newCode = [];	×
111	const code = ex.target_code.split(' ');	×
112	for (let i = 0; i < code.length; i++) {	×
113	const token = code[i];	×
114	if (token === '"') {	×
115	inString = !inString;	×
116	newCode.push(token);	×
117	continue;	×
118	}
119	if (inString) {	×
120	newCode.push(token);	×
121	continue;	×
122	}
123	if (token in renameMap) {	×
124	newCode.push(renameMap[token]);	×
125
126	// if we renamed a number into a measure, skip the unit
127	if (token.startsWith('NUMBER_') && renameMap[token].startsWith('MEASURE_'))	×
128	i++;	×
129	}
130
131	const match = ENTITY_MATCH_REGEX.exec(token);	×
132	if (match === null) {	×
133	newCode.push(token);	×
134	continue;	×
135	}
136
137	let [,type,] = match;	×
138
139	// collapse NUMBER_ followed by unit into measure or duration
140	if (type === 'NUMBER' && i < code.length-1 && code[i+1].startsWith('unit:')) {	×
141	const unit = code[i+1].substring('unit:'.length);	×
NEW 142	const baseunit = new ThingTalk.Type.Measure(unit).unit;	×
143	if (baseunit === 'ms')	×
144	type = 'DURATION';	×
145	else
146	type = 'MEASURE_' + baseunit;	×
147	// skip the unit
148	i++;	×
149	}
150
151	let newIdx;
152	if (type in countMap)	×
153	newIdx = countMap[type] + 1;	×
154	else
155	newIdx = 0;	×
156	countMap[type] = newIdx;	×
157
158	const newToken = type + '_' + newIdx;	×
159	if (newToken !== token)	×
160	anyRename = true;	×
161	renameMap[token] = newToken;	×
162	newCode.push(newToken);	×
163	}
164	ex.target_code = newCode.join(' ');	×
165	if (!anyRename)	×
166	return;	×
167	ex.preprocessed = ex.preprocessed.split(' ').map((token) => {	×
168	if (token in renameMap)	×
169	return renameMap[token];	×
170	else
171	return token;	×
172	}).join(' ');
173	}
174
175	_write(ex, encoding, callback) {
176	/*let yes = false;
177	if (ex.target_code === 'now => @com.foradb.findBP param:edate:Date = end_of unit:week - DURATION_1 param:patient:String = QUOTED_STRING_0 param:sdate:Date = end_of unit:mon + DURATION_0 => notify')
178	yes = true;*/
179	this._normalizeEntities(ex);	×
180	//if (yes)
181	// console.log(ex);
182
183	if (this._buffer.has(ex.target_code))	×
184	this._buffer.get(ex.target_code).push(ex.preprocessed);	×
185	else
186	this._buffer.set(ex.target_code, [ex.preprocessed]);	×
187	callback();	×
188	}
189
190	_finish(callback) {
191	callback();	×
192	}
193
194	async read() {
195	await StreamUtils.waitFinish(this);	×
196	return this._buffer;	×
197	}
198	}
199
200	module.exports = {	51✔
201	initArgparse(subparsers) {
202	const parser = subparsers.add_parser('sample', {	51✔
203	add_help: true,
204	description: "Choose which sentences to paraphrase, given a synthetic set."
205	});
206	parser.add_argument('-o', '--output', {	51✔
207	required: true,
208	type: fs.createWriteStream
209	});
210	parser.add_argument('--thingpedia', {	51✔
211	required: true,
212	help: 'Path to JSON file containing signature, type and mixin definitions.'
213	});
214	parser.add_argument('--constants', {	51✔
215	required: true,
216	help: 'TSV file containing constant values to use.'
217	});
218	parser.add_argument('input_file', {	51✔
219	nargs: '+',
220	type: maybeCreateReadStream,
221	help: 'Input datasets to augment (in TSV format); use - for standard input'
222	});
223	parser.add_argument('-l', '--locale', {	51✔
224	default: 'en-US',
225	help: `BGP 47 locale tag of the natural language being processed (defaults to en-US).`
226	});
227	parser.add_argument('--contextual', {	51✔
228	action: 'store_true',
229	help: 'Process a contextual dataset.',
230	default: false
231	});
232	parser.add_argument('--context-source', {	51✔
233	help: 'Source dataset from where contexts were extracted; used to choose a context sentence for each context.',
234	});
235	parser.add_argument('--sampling-strategy', {	51✔
236	required: false,
237	choices: ['byCode', 'bySentence', 'bySignature'],
238	help: 'Which sampling strategy to use (defaults: bySignature).'
239	});
240	parser.add_argument('--sampling-control', {	51✔
241	required: false,
242	help: 'TSV file controlling sampling based on functions in the programs. Defaults to treating all functions equally.'
243	});
244	parser.add_argument('--compound-only', {	51✔
245	help: 'Keep only compound programs. (False if omitted)',
246	action: 'store_true'
247	});
248
249	parser.add_argument('--debug', {	51✔
250	action: 'store_true',
251	help: 'Enable debugging.',
252	default: true
253	});
254	parser.add_argument('--no-debug', {	51✔
255	action: 'store_false',
256	dest: 'debug',
257	help: 'Disable debugging.',
258	});
259	parser.add_argument('--random-seed', {	51✔
260	default: 'almond is awesome',
261	help: 'Random seed'
262	});
263	},
264
265	async execute(args) {
266	let contexts;
267	if (args.contextual) {	1!
268	if (!args.context_source)	×
269	throw new Error(`--context-source is required if --contextual`);	×
270
271	contexts = await readAllLines([fs.createReadStream(args.context_source)])	×
272	.pipe(new DatasetParser({ preserveId: true }))
273	.pipe(new ContextSourceLoader())
274	.read();
275	}
276
277	const constants = await parseConstantFile(args.locale, args.constants);	1✔
278	const [functionBlackList, deviceBlackList, functionHighValueList, functionWhiteList, deviceWhiteList] =
279	await parseSamplingControlFile(args.sampling_control);	1✔
280
281	const tpClient = new Tp.FileClient(args.locale, args.thingpedia, null);	1✔
282	const schemaRetriever = new ThingTalk.SchemaRetriever(tpClient, null, !args.debug);	1✔
283
284	const options = {	1✔
285	rng: seedrandom.alea(args.random_seed),
286	locale: args.locale,
287
288	samplingStrategy: args.sampling_strategy,
289	functionBlackList,
290	deviceBlackList,
291	functionHighValueList,
292	functionWhiteList,
293	deviceWhiteList,
294	contexts,
295
296	compoundOnly: !!args.compound_only,
297	debug: args.debug
298	};
299
300	readAllLines(args.input_file)	1✔
301	.pipe(new DatasetParser({ contextual: args.contextual, preserveId: true }))
302	.pipe(new SentenceSampler(schemaRetriever, constants, options))
303	.pipe(csvstringify({ header: true, delimiter: '\t' }))
304	.pipe(args.output);
305
306	return new Promise((resolve, reject) => {	1✔
307	args.output.on('finish', resolve);	1✔
308	args.output.on('error', reject);	1✔
309	});
310	}
311	};

stanford-oval / genie-toolkit / 1340

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous