stanford-oval / genie-toolkit / 833

pending completion

Build # 833

Build Type

Pull #173

travis-ci-com

Committed by

web-flow

Commit Message

Merge f3dbc1a9c into f39778784

Pull Request Pull Request #173: Wip/schemaorg automation2

Run Details

1358 of 5151 branches covered (26.36%)

Branch coverage included in aggregate %.

82 of 359 new or added lines in 12 files covered. (22.84%)

23 existing lines in 7 files now uncovered.

4131 of 10096 relevant lines covered (40.92%)

134454.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.88

/tool/lib/file_parameter_provider.js

// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
//
// See COPYING for details
"use strict";

const util = require('util');
const fs = require('fs');
const byline = require('byline');
const csvparse = require('csv-parse');
const path = require('path');

// Load strings and entities from files
//
// Strings are loaded from the TSV files generated from
// https://almond.stanford.edu/thingpedia/strings/download/:x
//
// Entities are loaded from the JSON files returned from
// https://almond.stanford.edu/thingpedia/api/v3/entities/list/:x

module.exports = class FileParameterProvider {
    constructor(filename, param_locale) {
        this._filename = filename;
        this._paramLocale = param_locale || 'en-US';
        this._dirname = path.dirname(filename);
        this._paths = new Map;
    }

    async open() {
        const file = fs.createReadStream(this._filename);
        file.setEncoding('utf8');

        const input = byline(file);

        input.on('data', (line) => {
            if (/^\s*(#|$)/.test(line))
                return;

            const row = line.trim().split('\t');
            let stringOrEntity, locale, type, filepath;
            if (row.length === 4) {
                [stringOrEntity, locale, type, filepath] = row;
            } else {
                locale = 'en-US';
                [stringOrEntity, type, filepath] = row;
            }
            if (stringOrEntity !== 'string' && stringOrEntity !== 'entity')
                throw new Error(`Invalid syntax: ${line}`);
            if (locale === this._paramLocale)
                this._paths.set(stringOrEntity + '+' + type, path.resolve(this._dirname, filepath));
        });

        return new Promise((resolve, reject) => {
            input.on('end', resolve);
            input.on('error', reject);
        });
    }

    async close() {
    }

    async _getStrings(stringType) {
        const filepath = this._paths.get('string+' + stringType);
        if (!filepath)
            return [];

        const strings = [];
        const input = fs.createReadStream(filepath)
            .pipe(csvparse({ delimiter: '\t', relax: true }));

        input.on('data', (line) => {
            let value, preprocessed, weight;
            value = line[0];
            if (line.length === 1) {
                preprocessed = line[0];
                weight = 1.0;
            } else if (line.length === 2) {
                if (isFinite(+line[1])) {
                    preprocessed = line[0];
                    weight = line[1];
                } else {
                    preprocessed = line[1];
                    weight = 1.0;
                }
            } else {
                preprocessed = line[1];
                weight = parseFloat(line[2]) || 1.0;
            }
            if (!(weight > 0.0))
                weight = 1.0;

            strings.push({ value, preprocessed, weight });
        });

        return new Promise((resolve, reject) => {
            input.on('end', () => {
                if (strings.length === 0)
                    console.log('actually no values for', stringType, filepath);
                resolve(strings);
            });
            input.on('error', reject);
        });
    }

    async _getEntities(stringType) {
        const filepath = this._paths.get('entity+' + stringType);
        if (!filepath)
            return [];

        const parsed = JSON.parse(await util.promisify(fs.readFile)(filepath));
        return parsed.data.map((e) => {
            return { preprocessed: e.canonical, weight: 1.0, value:e.value, name:e.name };
        });
    }

    get(valueListType, valueListName) {
        switch (valueListType) {
        case 'string':
            return this._getStrings(valueListName);
        case 'entity':
            return this._getEntities(valueListName);
        default:
            throw new TypeError(`Unexpected value list type ${valueListType}`);
        }
    }
};

1	// -- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of Genie
4	//
5	// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
8	//
9	// See COPYING for details
10	"use strict";
11
12	const util = require('util');	52✔
13	const fs = require('fs');	52✔
14	const byline = require('byline');	52✔
15	const csvparse = require('csv-parse');	52✔
16	const path = require('path');	52✔
17
18	// Load strings and entities from files
19	//
20	// Strings are loaded from the TSV files generated from
21	// https://almond.stanford.edu/thingpedia/strings/download/:x
22	//
23	// Entities are loaded from the JSON files returned from
24	// https://almond.stanford.edu/thingpedia/api/v3/entities/list/:x
25
26	module.exports = class FileParameterProvider {	52✔
27	constructor(filename, param_locale) {
28	this._filename = filename;	4✔
29	this._paramLocale = param_locale \|\| 'en-US';	4!
30	this._dirname = path.dirname(filename);	4✔
31	this._paths = new Map;	4✔
32	}
33
34	async open() {
35	const file = fs.createReadStream(this._filename);	4✔
36	file.setEncoding('utf8');	4✔
37
38	const input = byline(file);	4✔
39
40	input.on('data', (line) => {	4✔
41	if (/^\s*(#\|$)/.test(line))	328✔
42	return;	30✔
43
44	const row = line.trim().split('\t');	298✔
45	let stringOrEntity, locale, type, filepath;
46	if (row.length === 4) {	298!
47	[stringOrEntity, locale, type, filepath] = row;	298✔
48	} else {
NEW 49	locale = 'en-US';	×
NEW 50	[stringOrEntity, type, filepath] = row;	×
51	}
52	if (stringOrEntity !== 'string' && stringOrEntity !== 'entity')	298!
53	throw new Error(`Invalid syntax: ${line}`);	×
54	if (locale === this._paramLocale)	298!
55	this._paths.set(stringOrEntity + '+' + type, path.resolve(this._dirname, filepath));	298✔
56	});
57
58	return new Promise((resolve, reject) => {	4✔
59	input.on('end', resolve);	4✔
60	input.on('error', reject);	4✔
61	});
62	}
63
64	async close() {
65	}
66
67	async _getStrings(stringType) {
68	const filepath = this._paths.get('string+' + stringType);	22✔
69	if (!filepath)	22✔
70	return [];	4✔
71
72	const strings = [];	18✔
73	const input = fs.createReadStream(filepath)	18✔
74	.pipe(csvparse({ delimiter: '\t', relax: true }));
75
76	input.on('data', (line) => {	18✔
77	let value, preprocessed, weight;
78	value = line[0];	1,232✔
79	if (line.length === 1) {	1,232!
80	preprocessed = line[0];	×
81	weight = 1.0;	×
82	} else if (line.length === 2) {	1,232!
83	if (isFinite(+line[1])) {	×
84	preprocessed = line[0];	×
85	weight = line[1];	×
86	} else {
87	preprocessed = line[1];	×
88	weight = 1.0;	×
89	}
90	} else {
91	preprocessed = line[1];	1,232✔
92	weight = parseFloat(line[2]) \|\| 1.0;	1,232!
93	}
94	if (!(weight > 0.0))	1,232!
95	weight = 1.0;	×
96
97	strings.push({ value, preprocessed, weight });	1,232✔
98	});
99
100	return new Promise((resolve, reject) => {	18✔
101	input.on('end', () => {	18✔
102	if (strings.length === 0)	18!
103	console.log('actually no values for', stringType, filepath);	×
104	resolve(strings);	18✔
105	});
106	input.on('error', reject);	18✔
107	});
108	}
109
110	async _getEntities(stringType) {
111	const filepath = this._paths.get('entity+' + stringType);	6✔
112	if (!filepath)	6!
113	return [];	×
114
115	const parsed = JSON.parse(await util.promisify(fs.readFile)(filepath));	6✔
116	return parsed.data.map((e) => {	6✔
117	return { preprocessed: e.canonical, weight: 1.0, value:e.value, name:e.name };	92✔
118	});
119	}
120
121	get(valueListType, valueListName) {
122	switch (valueListType) {	28!
123	case 'string':
124	return this._getStrings(valueListName);	22✔
125	case 'entity':
126	return this._getEntities(valueListName);	6✔
127	default:
128	throw new TypeError(`Unexpected value list type ${valueListType}`);	×
129	}
130	}
131	};

stanford-oval / genie-toolkit / 833

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous