2456

pending completion

Build # 2456

Build Type

Pull #817

travis-ci-com

Committed by

web-flow

Commit Message

Merge 2a8d1a019 into 72c3186f4

Pull Request Pull Request #817: PO tests

Run Details

10245 of 20789 branches covered (49.28%)

Branch coverage included in aggregate %.

4 of 4 new or added lines in 2 files covered. (100.0%)

3 existing lines in 1 file now uncovered.

24834 of 41461 relevant lines covered (59.9%)

437857.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.43

/tool/translate-schema-annotations.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Mehrad Moradshahi <mehrad@cs.stanford.edu>

import * as argparse from 'argparse';
import * as gettextParser from 'gettext-parser';
import assert from 'assert';
import * as fs from 'fs';
import { promises as pfs } from 'fs';
import * as ThingTalk from 'thingtalk';
import * as StreamUtils from "../lib/utils/stream-utils";

export function initArgparse(subparsers : argparse.SubParser) {
    const parser = subparsers.add_parser('translate-schema-annotations', {
        add_help: true,
        description: "Translate a thingpedia file's annotations to another language."
    });
    parser.add_argument('--po-file', {
        required: true,
        help: 'PO file containing original and translated strings'
    });
    parser.add_argument('input_file', {
        help: 'Input Thingpedia file to read'
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
}

const translationDict = new Map<string, string>();

function* returnAllKeys(key : string, str : unknown) : any {
    if (typeof str === 'string') {
        yield key;
    } else if (Array.isArray(str)) {
        for (let i = 0; i < str.length; i++)
            yield* returnAllKeys(`${key}[${i}]`, str[i]);
    } else if (typeof str === 'object' && str !== null) {
        for (const subkey in str) {
            if (subkey === 'type' || subkey === 'default')
                continue;
            yield* returnAllKeys(`${key};${subkey}`, str[subkey as keyof typeof str]);
        }
    } else {
        yield null;
    }
}

function processEnum(arg : any, iterKey : string,) {
    const valList : Array<string | undefined> = [];
    assert(typeof arg.type === 'object');
    for (const k of returnAllKeys(iterKey, arg.type)) {
        if (k === null)
            continue;
        valList.push(translationDict.get(k));
    }

    const keys : string[] = arg.type.entries;
    assert(keys.length === valList.length);

    const valueEnum = Object.fromEntries(keys.map((_, i) => [keys[i], [valList[i]]]));

    if (typeof arg.metadata['canonical'] === 'string' || Array.isArray(arg.metadata['canonical']))
        arg.metadata['canonical'] = { 'base': arg.metadata['canonical'], 'value_enum': valueEnum };
    else
        arg.metadata['canonical']['value_enum'] = valueEnum;

}

function process(obj : any, iterKey : string) {
    for (const key in obj) {
        for (const k of returnAllKeys(iterKey + `.${key}`, obj[key])) {
            if (k === null)
                continue;
            if (Array.isArray(obj[key])) {
                let subKey = k.split('.').slice(-1);
                // TODO fix
                if (subKey.length !== 1) {
                    console.log(subKey);
                    continue;
                }
                subKey = subKey[0];
                const [, , id_] = /(.+?)\[(\d+)\]/.exec(subKey)!;
                obj[key][id_] = translationDict.get(k);
            } else if (typeof obj[key] === 'object') {
                let subKey = k.split(';').slice(1);
                // TODO fix
                if (subKey.length !== 1)
                    console.log(subKey);
                subKey = subKey[0];
                const [, substr, id_] = /(.+?)\[(\d+)\]/.exec(subKey)!;
                obj[key][substr][id_] = translationDict.get(k);
            } else {
                obj[key] = translationDict.get(k);
            }
        }
    }

}

export async function execute(args : any) {
    const code = (await pfs.readFile(args.input_file)).toString();
    const parsed = ThingTalk.Syntax.parse(code, ThingTalk.Syntax.SyntaxType.Normal, {
        locale: 'en-US',
        timezone: 'UTC'
    });
    assert(parsed instanceof ThingTalk.Ast.Library);

    const pofile = gettextParser.po.parse(await pfs.readFile(args.po_file, { encoding: 'utf8' }));

    // get translations for the default context
    const poObjects = pofile.translations[''];
    for (const msgid in poObjects) {
        const entry : gettextParser.GetTextTranslation = poObjects[msgid];
        const refKeys : string[]|undefined = entry.comments?.reference?.split('\n');
        const extKeys : string[]|undefined = entry.comments?.extracted?.split('\n');
        for (const keys of [refKeys, extKeys]) {
            if (!keys)
                continue;
            for (const k of keys) {
                assert(entry.msgstr.length === 1);
                translationDict.set(k, entry.msgstr[0]);
            }
        }
    }

    // handle manifest
    for (const _class of parsed.classes) {
        for (const key in _class.metadata)
            _class.metadata[key] = translationDict.get(key);
        for (const what of ['queries', 'actions'] as Array<'queries' | 'actions'>) {
            for (const name in _class[what]) {
                process(_class[what][name].metadata, `${what}.${name}`);
                for (const argname of _class[what][name].args) {
                    const arg = _class[what][name].getArgument(argname)!;
                    process(arg.metadata, `${what}.${name}.args.${argname}`);

                    if (arg.type.isEnum)
                        processEnum(arg, `${what}.${name}.args.${argname}.enum`);
                }
            }
        }
    }

    // handle dataset
    for (const _class of parsed.datasets) {
        for (const [ex_id, ex] of _class.examples.entries()) {
            for (const [uttr_id, _] of ex.utterances.entries()) {
                const key = `${_class.name}.${ex_id}.${uttr_id}`;
                ex.utterances[uttr_id] = translationDict.get(key)!;
            }
        }
    }

    if (args.output) {
        args.output.end(parsed.prettyprint());
        await StreamUtils.waitFinish(args.output);
    }

}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2021 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Mehrad Moradshahi <mehrad@cs.stanford.edu>
20
21	import * as argparse from 'argparse';
22	import * as gettextParser from 'gettext-parser';	103✔
23	import assert from 'assert';	103✔
24	import * as fs from 'fs';	103✔
25	import { promises as pfs } from 'fs';	103✔
26	import * as ThingTalk from 'thingtalk';	103✔
27	import * as StreamUtils from "../lib/utils/stream-utils";	103✔
28
29	export function initArgparse(subparsers : argparse.SubParser) {	103✔
30	const parser = subparsers.add_parser('translate-schema-annotations', {	103✔
31	add_help: true,
32	description: "Translate a thingpedia file's annotations to another language."
33	});
34	parser.add_argument('--po-file', {	103✔
35	required: true,
36	help: 'PO file containing original and translated strings'
37	});
38	parser.add_argument('input_file', {	103✔
39	help: 'Input Thingpedia file to read'
40	});
41	parser.add_argument('-o', '--output', {	103✔
42	required: true,
43	type: fs.createWriteStream
44	});
45	}
46
47	const translationDict = new Map<string, string>();	103✔
48
49	function* returnAllKeys(key : string, str : unknown) : any {
50	if (typeof str === 'string') {	213✔
51	yield key;	124✔
52	} else if (Array.isArray(str)) {	89✔
53	for (let i = 0; i < str.length; i++)	33✔
54	yield* returnAllKeys(`${key}[${i}]`, str[i]);	101✔
55	} else if (typeof str === 'object' && str !== null) {	56✔
56	for (const subkey in str) {	11✔
57	if (subkey === 'type' \|\| subkey === 'default')	76✔
58	continue;	7✔
59	yield* returnAllKeys(`${key};${subkey}`, str[subkey as keyof typeof str]);	69✔
60	}
61	} else {
62	yield null;	45✔
63	}
64	}
65
66	function processEnum(arg : any, iterKey : string,) {
67	const valList : Array<string \| undefined> = [];	3✔
68	assert(typeof arg.type === 'object');	3✔
69	for (const k of returnAllKeys(iterKey, arg.type)) {	3✔
70	if (k === null)	69✔
71	continue;	45✔
72	valList.push(translationDict.get(k));	24✔
73	}
74
75	const keys : string[] = arg.type.entries;	3✔
76	assert(keys.length === valList.length);	3✔
77
78	const valueEnum = Object.fromEntries(keys.map((_, i) => [keys[i], [valList[i]]]));	24✔
79
80	if (typeof arg.metadata['canonical'] === 'string' \|\| Array.isArray(arg.metadata['canonical']))	3!
81	arg.metadata['canonical'] = { 'base': arg.metadata['canonical'], 'value_enum': valueEnum };	3✔
82	else
UNCOV 83	arg.metadata['canonical']['value_enum'] = valueEnum;	×
84
85	}
86
87	function process(obj : any, iterKey : string) {
88	for (const key in obj) {	28✔
89	for (const k of returnAllKeys(iterKey + `.${key}`, obj[key])) {	40✔
90	if (k === null)	100!
91	continue;	×
92	if (Array.isArray(obj[key])) {	100✔
93	let subKey = k.split('.').slice(-1);	35✔
94	// TODO fix
95	if (subKey.length !== 1) {	35!
UNCOV 96	console.log(subKey);	×
97	continue;	×
98	}
99	subKey = subKey[0];	35✔
100	const [, , id_] = /(.+?)\[(\d+)\]/.exec(subKey)!;	35✔
101	obj[key][id_] = translationDict.get(k);	35✔
102	} else if (typeof obj[key] === 'object') {	65✔
103	let subKey = k.split(';').slice(1);	42✔
104	// TODO fix
105	if (subKey.length !== 1)	42!
UNCOV 106	console.log(subKey);	×
107	subKey = subKey[0];	42✔
108	const [, substr, id_] = /(.+?)\[(\d+)\]/.exec(subKey)!;	42✔
109	obj[key][substr][id_] = translationDict.get(k);	42✔
110	} else {
111	obj[key] = translationDict.get(k);	23✔
112	}
113	}
114	}
115
116	}
117
118	export async function execute(args : any) {	103✔
119	const code = (await pfs.readFile(args.input_file)).toString();	2✔
120	const parsed = ThingTalk.Syntax.parse(code, ThingTalk.Syntax.SyntaxType.Normal, {	2✔
121	locale: 'en-US',
122	timezone: 'UTC'
123	});
124	assert(parsed instanceof ThingTalk.Ast.Library);	2✔
125
126	const pofile = gettextParser.po.parse(await pfs.readFile(args.po_file, { encoding: 'utf8' }));	2✔
127
128	// get translations for the default context
129	const poObjects = pofile.translations[''];	2✔
130	for (const msgid in poObjects) {	2✔
131	const entry : gettextParser.GetTextTranslation = poObjects[msgid];	3,856✔
132	const refKeys : string[]\|undefined = entry.comments?.reference?.split('\n');	3,856!
133	const extKeys : string[]\|undefined = entry.comments?.extracted?.split('\n');	3,856!
134	for (const keys of [refKeys, extKeys]) {	3,856✔
135	if (!keys)	7,712✔
136	continue;	2,694✔
137	for (const k of keys) {	5,018✔
138	assert(entry.msgstr.length === 1);	6,738✔
139	translationDict.set(k, entry.msgstr[0]);	6,738✔
140	}
141	}
142	}
143
144	// handle manifest
145	for (const _class of parsed.classes) {	2✔
146	for (const key in _class.metadata)	1✔
147	_class.metadata[key] = translationDict.get(key);	2✔
148	for (const what of ['queries', 'actions'] as Array<'queries' \| 'actions'>) {	1✔
149	for (const name in _class[what]) {	2✔
150	process(_class[what][name].metadata, `${what}.${name}`);	4✔
151	for (const argname of _class[what][name].args) {	4✔
152	const arg = _class[what][name].getArgument(argname)!;	24✔
153	process(arg.metadata, `${what}.${name}.args.${argname}`);	24✔
154
155	if (arg.type.isEnum)	24✔
156	processEnum(arg, `${what}.${name}.args.${argname}.enum`);	3✔
157	}
158	}
159	}
160	}
161
162	// handle dataset
163	for (const _class of parsed.datasets) {	2✔
164	for (const [ex_id, ex] of _class.examples.entries()) {	1✔
165	for (const [uttr_id, _] of ex.utterances.entries()) {	22✔
166	const key = `${_class.name}.${ex_id}.${uttr_id}`;	29✔
167	ex.utterances[uttr_id] = translationDict.get(key)!;	29✔
168	}
169	}
170	}
171
172	if (args.output) {	2!
173	args.output.end(parsed.prettyprint());	2✔
174	await StreamUtils.waitFinish(args.output);	2✔
175	}
176
177	}

stanford-oval / genie-toolkit / 2456

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous