2372

pending completion

Build # 2372

Build Type

Pull #783

travis-ci-com

Committed by

web-flow

Commit Message

Merge 6fd65a803 into 9d5cafabf

Pull Request Pull Request #783: Part 1 of fixing timezone issues

Run Details

10150 of 20529 branches covered (49.44%)

Branch coverage included in aggregate %.

107 of 138 new or added lines in 45 files covered. (77.54%)

11 existing lines in 7 files now uncovered.

24284 of 40651 relevant lines covered (59.74%)

338917.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.01

/tool/download-entity-values.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2021 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import * as ThingTalk from 'thingtalk';
import * as argparse from 'argparse';
import * as fs from 'fs';
import * as Tp from 'thingpedia';
import * as path from 'path';
import * as util from 'util';

import * as StreamUtils from '../lib/utils/stream-utils';
import ProgressBar from './lib/progress_bar';
import { getConfig, DEFAULT_THINGPEDIA_URL } from './lib/argutils';

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('download-entity-values', {
        add_help: true,
        description: "Download one or more entity dataset from Thingpedia."
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the natural language to download the snapshot for (defaults to 'en-US', American English)`
    });
    parser.add_argument('--thingpedia-url', {
        required: false,
        help: `base URL of Thingpedia server to contact; defaults to '${DEFAULT_THINGPEDIA_URL}'`
    });
    parser.add_argument('--developer-key', {
        required: false,
        default: '',
        help: `developer key to use when contacting Thingpedia`
    });
    parser.add_argument('-d', '--output-dir', {
        required: true,
    });
    parser.add_argument('--type', {
        required: false,
        action: 'append',
        default: [],
        help: `identifier of the Thingpedia entity datasets to download (if omitted, all entities are downloaded)`
    });
    parser.add_argument('--manifest', {
        required: false,
        help: `write a parameter dataset manifest to this location (suitable for Genie)`
    });
    parser.add_argument('--append-manifest', {
        required: false,
        action: 'store_true',
        help: `append to the manifest instead of replacing`
    });

    parser.add_argument('--debug', {
        action: 'store_true',
        help: 'Enable debugging.',
        default: false
    });
    parser.add_argument('--no-debug', {
        action: 'store_false',
        dest: 'debug',
        help: 'Disable debugging.',
    });
}

interface EntityTypeRecord {
    type : string;
    name : string;
    is_well_known : boolean|number;
    has_ner_support : boolean|number;
}

export async function execute(args : any) {
    if (!args.thingpedia_url)
        args.thingpedia_url = await getConfig('thingpedia.url', process.env.THINGPEDIA_URL || DEFAULT_THINGPEDIA_URL);
    if (!args.developer_key)
        args.developer_key = await getConfig('thingpedia.developer-key', process.env.THINGPEDIA_DEVELOPER_KEY || null);

    await util.promisify(fs.mkdir)(args.output_dir, { recursive: true });

    if (args.type.length === 0) {
        let listUrl = `${args.thingpedia_url}/api/v3/entities/all?locale=${args.locale}&thingtalk_version=${ThingTalk.version}`;
        if (args.developer_key)
            listUrl += '&developer_key=' + args.developer_key;

        args.type = (JSON.parse(await Tp.Helpers.Http.get(listUrl, { accept: 'application/json' }))
            .data as EntityTypeRecord[]).filter((e) => !e.is_well_known && e.has_ner_support).map((e) => e.type);
    }

    let progbar;
    if (!args.debug) {
        progbar = new ProgressBar(args.type.length);
        // issue an update now to show the progress bar
        progbar.update(0);
    }

    let manifest;
    if (args.manifest)
        manifest = fs.createWriteStream(args.manifest, { flags: args.append_manifest ? 'a' : 'w' });

    let progress = 0;
    for (const type of args.type) {
        let url = `${args.thingpedia_url}/api/v3/entities/list/${type}?locale=${args.locale}&thingtalk_version=${ThingTalk.version}`;
        if (args.developer_key)
            url += '&developer_key=' + args.developer_key;

        const outputpath = path.resolve(args.output_dir, type + '.json');
        const output = fs.createWriteStream(outputpath);

        if (manifest)
            manifest.write(`entity\t${args.locale}\t${type}\t${path.relative(path.dirname(args.manifest), outputpath)}\n`);

        (await Tp.Helpers.Http.getStream(url)).pipe(output);
        await StreamUtils.waitFinish(output);

        progress++;
        if (progbar)
            progbar.update(progress);
    }

    if (manifest) {
        manifest.end();
        await StreamUtils.waitFinish(manifest);
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019-2021 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import * as ThingTalk from 'thingtalk';	88✔
22	import * as argparse from 'argparse';
23	import * as fs from 'fs';	88✔
24	import * as Tp from 'thingpedia';	88✔
25	import * as path from 'path';	88✔
26	import * as util from 'util';	88✔
27
28	import * as StreamUtils from '../lib/utils/stream-utils';	88✔
29	import ProgressBar from './lib/progress_bar';	88✔
30	import { getConfig, DEFAULT_THINGPEDIA_URL } from './lib/argutils';	88✔
31
32	export function initArgparse(subparsers : argparse.SubParser) {	88✔
33	const parser = subparsers.add_parser('download-entity-values', {	88✔
34	add_help: true,
35	description: "Download one or more entity dataset from Thingpedia."
36	});
37	parser.add_argument('-l', '--locale', {	88✔
38	required: false,
39	default: 'en-US',
40	help: `BGP 47 locale tag of the natural language to download the snapshot for (defaults to 'en-US', American English)`
41	});
42	parser.add_argument('--thingpedia-url', {	88✔
43	required: false,
44	help: `base URL of Thingpedia server to contact; defaults to '${DEFAULT_THINGPEDIA_URL}'`
45	});
46	parser.add_argument('--developer-key', {	88✔
47	required: false,
48	default: '',
49	help: `developer key to use when contacting Thingpedia`
50	});
51	parser.add_argument('-d', '--output-dir', {	88✔
52	required: true,
53	});
54	parser.add_argument('--type', {	88✔
55	required: false,
56	action: 'append',
57	default: [],
58	help: `identifier of the Thingpedia entity datasets to download (if omitted, all entities are downloaded)`
59	});
60	parser.add_argument('--manifest', {	88✔
61	required: false,
62	help: `write a parameter dataset manifest to this location (suitable for Genie)`
63	});
64	parser.add_argument('--append-manifest', {	88✔
65	required: false,
66	action: 'store_true',
67	help: `append to the manifest instead of replacing`
68	});
69
70	parser.add_argument('--debug', {	88✔
71	action: 'store_true',
72	help: 'Enable debugging.',
73	default: false
74	});
75	parser.add_argument('--no-debug', {	88✔
76	action: 'store_false',
77	dest: 'debug',
78	help: 'Disable debugging.',
79	});
80	}
81
82	interface EntityTypeRecord {
83	type : string;
84	name : string;
85	is_well_known : boolean\|number;
86	has_ner_support : boolean\|number;
87	}
88
89	export async function execute(args : any) {	88✔
90	if (!args.thingpedia_url)	4!
UNCOV 91	args.thingpedia_url = await getConfig('thingpedia.url', process.env.THINGPEDIA_URL \|\| DEFAULT_THINGPEDIA_URL);	×
92	if (!args.developer_key)	4!
UNCOV 93	args.developer_key = await getConfig('thingpedia.developer-key', process.env.THINGPEDIA_DEVELOPER_KEY \|\| null);	×
94
95	await util.promisify(fs.mkdir)(args.output_dir, { recursive: true });	4✔
96
97	if (args.type.length === 0) {	4!
98	let listUrl = `${args.thingpedia_url}/api/v3/entities/all?locale=${args.locale}&thingtalk_version=${ThingTalk.version}`;	4✔
99	if (args.developer_key)	4!
100	listUrl += '&developer_key=' + args.developer_key;	4✔
101
102	args.type = (JSON.parse(await Tp.Helpers.Http.get(listUrl, { accept: 'application/json' }))	4✔
103	.data as EntityTypeRecord[]).filter((e) => !e.is_well_known && e.has_ner_support).map((e) => e.type);	948✔
104	}
105
106	let progbar;
107	if (!args.debug) {	4!
108	progbar = new ProgressBar(args.type.length);	4✔
109	// issue an update now to show the progress bar
110	progbar.update(0);	4✔
111	}
112
113	let manifest;
114	if (args.manifest)	4!
115	manifest = fs.createWriteStream(args.manifest, { flags: args.append_manifest ? 'a' : 'w' });	4!
116
117	let progress = 0;	4✔
118	for (const type of args.type) {	4✔
119	let url = `${args.thingpedia_url}/api/v3/entities/list/${type}?locale=${args.locale}&thingtalk_version=${ThingTalk.version}`;	460✔
120	if (args.developer_key)	460!
121	url += '&developer_key=' + args.developer_key;	460✔
122
123	const outputpath = path.resolve(args.output_dir, type + '.json');	460✔
124	const output = fs.createWriteStream(outputpath);	460✔
125
126	if (manifest)	460!
127	manifest.write(`entity\t${args.locale}\t${type}\t${path.relative(path.dirname(args.manifest), outputpath)}\n`);	460✔
128
129	(await Tp.Helpers.Http.getStream(url)).pipe(output);	460✔
130	await StreamUtils.waitFinish(output);	460✔
131
132	progress++;	460✔
133	if (progbar)	460!
134	progbar.update(progress);	460✔
135	}
136
137	if (manifest) {	4!
138	manifest.end();	4✔
139	await StreamUtils.waitFinish(manifest);	4✔
140	}
141	}

stanford-oval / genie-toolkit / 2372

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous