1180

pending completion

Build # 1180

Build Type

Pull #287

travis-ci-com

Committed by

web-flow

Commit Message

Merge be76e2d0b into c367506f0

Pull Request Pull Request #287: Optimize parsing ThingTalk code

Run Details

4865 of 6706 branches covered (72.55%)

Branch coverage included in aggregate %.

7 of 7 new or added lines in 1 file covered. (100.0%)

1 existing line in 1 file now uncovered.

10663 of 13113 relevant lines covered (81.32%)

48800.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.58

/lib/nn-syntax/lexer.ts

// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of ThingTalk
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>

import Type from '../type';

import { SourceRange } from '../utils/source_locations';
import { AnyEntity, GenericEntity, EntityMap, EntityResolver } from '../entities';

interface FunctionToken {
    kind : string;
    channel : string;
}

interface ContextRefToken {
    name : string;
    type : Type;
}

interface GenericEntityToken {
    name : string;
    value : string;
    type : string;
}

type TokenValue = AnyEntity | GenericEntityToken | FunctionToken | ContextRefToken;

class TokenWrapper {
    location : SourceRange;

    constructor(public token : string,
                public value : TokenValue,
                location ?: number) {
        this.location = {
            start: {
                offset: 0,
                line: 0,
                column: 0,
                token: location
            },
            end: {
                offset: 0,
                line: 0,
                column: 0,
                token: location
            }
        };
    }

    toString() : string {
        return this.token;
    }
}

function isEntity(token : string) : boolean {
    // an entity is a token that starts with a fully uppercase word followed by _, followed by other stuff
    return /^[A-Z]+_/.test(token);
}


export default class SequenceLexer implements Iterator<TokenWrapper> {
    private _sequence : string[];
    private _entities : EntityResolver;
    private _i : number;
    private _lastfunction : string|null;
    private _lastparam : string|null;
    private _instring : boolean;

    constructor(sequence : Iterable<string>, entities : EntityResolver|EntityMap) {
        if (Array.isArray(sequence))
            this._sequence = sequence;
        else
            this._sequence = Array.from(sequence);

        if (typeof entities !== 'function') {
            this._entities = (next : string) : AnyEntity => {
                if (!(next in entities)) {
                    if (next.startsWith('SLOT_'))
                        return undefined;
                    throw new SyntaxError('Invalid entity ' + next + ', have ' + Object.keys(entities));
                }
                return entities[next];
            };
        } else {
            this._entities = entities;
        }

        this._i = 0;
        this._lastfunction = null;
        this._lastparam = null;
        this._instring = false;
    }

    next() : IteratorResult<TokenWrapper> {
        if (this._i >= this._sequence.length)
            return { done: true, value: undefined };

        const token = this._sequence[this._i++];
        let next : TokenWrapper|string = token;

        if (token === '"') {
            this._instring = !this._instring;
        } else if (this._instring) {
            next = new TokenWrapper('WORD', token, this._i);
        } else if (/^[0-9]+$/.test(next) && token !== '0' && token !== '1') {
            next = new TokenWrapper('LITERAL_INTEGER', parseInt(token));
        } else if (/^time:[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$/.test(token)) {
            // need to remove 'time:' prefix because parser.lr uses split(':')[0] for hour
            next = new TokenWrapper('LITERAL_TIME', token.replace('time:', ''));
        } else if (isEntity(token)) {
            // check if we have a unit next, to pass to the entity retriever
            let unit : string|null = null;
            // note that this._i has already been increased
            if (this._i < this._sequence.length && this._sequence[this._i].startsWith('unit:'))
                unit = this._sequence[this._i].substring('unit:'.length);

            // entity
            const entity = this._entities(token, this._lastparam, this._lastfunction, unit);
            const entityType = token.substring(0, token.lastIndexOf('_'));
            if (entityType.startsWith('GENERIC_ENTITY_')) {
                const generic = entity as GenericEntity;
                next = new TokenWrapper('GENERIC_ENTITY', {
                    value: generic.value,
                    display: generic.display,
                    type: entityType.substring('GENERIC_ENTITY_'.length)
                });
            } else if (entityType.startsWith('MEASURE_')) {
                next = new TokenWrapper('MEASURE', entity);
            } else {
                next = new TokenWrapper(entityType, entity);
            }
        } else if (token.startsWith('@')) {
            this._lastfunction = token;
            const lastPeriod = token.lastIndexOf('.');
            const kind = token.substring(1, lastPeriod);
            const channel = token.substring(lastPeriod+1);
            if (!kind || !channel)
                throw new Error('Invalid function ' + token);
            if (channel === '*')
                next = new TokenWrapper('CLASS_STAR', kind);
            else
                next = new TokenWrapper('FUNCTION', { kind, channel });
        } else if (next.startsWith('enum:')) {
            next = new TokenWrapper('ENUM', token.substring('enum:'.length));
        } else if (next.startsWith('param:')) {
            const [,paramname,] = next.split(':');
            this._lastparam = paramname;
            next = new TokenWrapper('PARAM_NAME', paramname);
        } else if (next.startsWith('attribute:')) {
            const [,paramname,] = next.split(':');
            this._lastparam = paramname;
            next = new TokenWrapper('ATTRIBUTE_NAME', paramname);
        } else if (next.startsWith('unit:$')) {
            next = new TokenWrapper('CURRENCY_CODE', token.substring('unit:$'.length));
        } else if (next.startsWith('unit:')) {
            next = new TokenWrapper('UNIT', token.substring('unit:'.length));
        } else if (next.startsWith('device:')) {
            next = new TokenWrapper('DEVICE', token.substring('device:'.length));
        } else if (next.startsWith('special:')) {
            next = new TokenWrapper('SPECIAL', token.substring('special:'.length));
        } else if (next.startsWith('context:')) {
            const withoutPrefix = token.substring('context:'.length);
            const colon = withoutPrefix.indexOf(':');
            const name = withoutPrefix.substring(0, colon);
            const type = Type.fromString(withoutPrefix.substring(colon+1));
            next = new TokenWrapper('CONTEXT_REF', { name, type });
        } else if (next.startsWith('^^')) {
            next = new TokenWrapper('ENTITY_TYPE', token.substring('^^'.length));
        }

        if (typeof next === 'string')
            next = new TokenWrapper(next, next);
        return { done: false, value: next };
    }
}

1	// -- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 --
2	//
3	// This file is part of ThingTalk
4	//
5	// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
6	//
7	// Licensed under the Apache License, Version 2.0 (the "License");
8	// you may not use this file except in compliance with the License.
9	// You may obtain a copy of the License at
10	//
11	// http://www.apache.org/licenses/LICENSE-2.0
12	//
13	// Unless required by applicable law or agreed to in writing, software
14	// distributed under the License is distributed on an "AS IS" BASIS,
15	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	// See the License for the specific language governing permissions and
17	// limitations under the License.
18	//
19	// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
20
21	import Type from '../type';	2✔
22
23	import { SourceRange } from '../utils/source_locations';
24	import { AnyEntity, GenericEntity, EntityMap, EntityResolver } from '../entities';
25
26	interface FunctionToken {
27	kind : string;
28	channel : string;
29	}
30
31	interface ContextRefToken {
32	name : string;
33	type : Type;
34	}
35
36	interface GenericEntityToken {
37	name : string;
38	value : string;
39	type : string;
40	}
41
42	type TokenValue = AnyEntity \| GenericEntityToken \| FunctionToken \| ContextRefToken;
43
44	class TokenWrapper {
45	location : SourceRange;
46
47	constructor(public token : string,	2,968✔
48	public value : TokenValue,	2,968✔
49	location ?: number) {
50	this.location = {	2,968✔
51	start: {
52	offset: 0,
53	line: 0,
54	column: 0,
55	token: location
56	},
57	end: {
58	offset: 0,
59	line: 0,
60	column: 0,
61	token: location
62	}
63	};
64	}
65
66	toString() : string {
UNCOV 67	return this.token;	×
68	}
69	}
70
71	function isEntity(token : string) : boolean {
72	// an entity is a token that starts with a fully uppercase word followed by _, followed by other stuff
73	return /^[A-Z]+_/.test(token);	2,876✔
74	}
75
76
77	export default class SequenceLexer implements Iterator<TokenWrapper> {	2✔
78	private _sequence : string[];
79	private _entities : EntityResolver;
80	private _i : number;
81	private _lastfunction : string\|null;
82	private _lastparam : string\|null;
83	private _instring : boolean;
84
85	constructor(sequence : Iterable<string>, entities : EntityResolver\|EntityMap) {
86	if (Array.isArray(sequence))	220!
87	this._sequence = sequence;	220✔
88	else
89	this._sequence = Array.from(sequence);	×
90
91	if (typeof entities !== 'function') {	220!
92	this._entities = (next : string) : AnyEntity => {	220✔
93	if (!(next in entities)) {	218!
94	if (next.startsWith('SLOT_'))	×
95	return undefined;	×
96	throw new SyntaxError('Invalid entity ' + next + ', have ' + Object.keys(entities));	×
97	}
98	return entities[next];	218✔
99	};
100	} else {
101	this._entities = entities;	×
102	}
103
104	this._i = 0;	220✔
105	this._lastfunction = null;	220✔
106	this._lastparam = null;	220✔
107	this._instring = false;	220✔
108	}
109
110	next() : IteratorResult<TokenWrapper> {
111	if (this._i >= this._sequence.length)	3,188✔
112	return { done: true, value: undefined };	220✔
113
114	const token = this._sequence[this._i++];	2,968✔
115	let next : TokenWrapper\|string = token;	2,968✔
116
117	if (token === '"') {	2,968✔
118	this._instring = !this._instring;	40✔
119	} else if (this._instring) {	2,928✔
120	next = new TokenWrapper('WORD', token, this._i);	24✔
121	} else if (/^[0-9]+$/.test(next) && token !== '0' && token !== '1') {	2,904✔
122	next = new TokenWrapper('LITERAL_INTEGER', parseInt(token));	18✔
123	} else if (/^time:[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$/.test(token)) {	2,886✔
124	// need to remove 'time:' prefix because parser.lr uses split(':')[0] for hour
125	next = new TokenWrapper('LITERAL_TIME', token.replace('time:', ''));	10✔
126	} else if (isEntity(token)) {	2,876✔
127	// check if we have a unit next, to pass to the entity retriever
128	let unit : string\|null = null;	218✔
129	// note that this._i has already been increased
130	if (this._i < this._sequence.length && this._sequence[this._i].startsWith('unit:'))	218✔
131	unit = this._sequence[this._i].substring('unit:'.length);	4✔
132
133	// entity
134	const entity = this._entities(token, this._lastparam, this._lastfunction, unit);	218✔
135	const entityType = token.substring(0, token.lastIndexOf('_'));	218✔
136	if (entityType.startsWith('GENERIC_ENTITY_')) {	218✔
137	const generic = entity as GenericEntity;	24✔
138	next = new TokenWrapper('GENERIC_ENTITY', {	24✔
139	value: generic.value,
140	display: generic.display,
141	type: entityType.substring('GENERIC_ENTITY_'.length)
142	});
143	} else if (entityType.startsWith('MEASURE_')) {	194✔
144	next = new TokenWrapper('MEASURE', entity);	4✔
145	} else {
146	next = new TokenWrapper(entityType, entity);	190✔
147	}
148	} else if (token.startsWith('@')) {	2,658✔
149	this._lastfunction = token;	256✔
150	const lastPeriod = token.lastIndexOf('.');	256✔
151	const kind = token.substring(1, lastPeriod);	256✔
152	const channel = token.substring(lastPeriod+1);	256✔
153	if (!kind \|\| !channel)	256!
154	throw new Error('Invalid function ' + token);	×
155	if (channel === '*')	256!
156	next = new TokenWrapper('CLASS_STAR', kind);	×
157	else
158	next = new TokenWrapper('FUNCTION', { kind, channel });	256✔
159	} else if (next.startsWith('enum:')) {	2,402✔
160	next = new TokenWrapper('ENUM', token.substring('enum:'.length));	18✔
161	} else if (next.startsWith('param:')) {	2,384✔
162	const [,paramname,] = next.split(':');	282✔
163	this._lastparam = paramname;	282✔
164	next = new TokenWrapper('PARAM_NAME', paramname);	282✔
165	} else if (next.startsWith('attribute:')) {	2,102✔
166	const [,paramname,] = next.split(':');	6✔
167	this._lastparam = paramname;	6✔
168	next = new TokenWrapper('ATTRIBUTE_NAME', paramname);	6✔
169	} else if (next.startsWith('unit:$')) {	2,096✔
170	next = new TokenWrapper('CURRENCY_CODE', token.substring('unit:$'.length));	2✔
171	} else if (next.startsWith('unit:')) {	2,094✔
172	next = new TokenWrapper('UNIT', token.substring('unit:'.length));	6✔
173	} else if (next.startsWith('device:')) {	2,088✔
174	next = new TokenWrapper('DEVICE', token.substring('device:'.length));	2✔
175	} else if (next.startsWith('special:')) {	2,086✔
176	next = new TokenWrapper('SPECIAL', token.substring('special:'.length));	4✔
177	} else if (next.startsWith('context:')) {	2,082✔
178	const withoutPrefix = token.substring('context:'.length);	2✔
179	const colon = withoutPrefix.indexOf(':');	2✔
180	const name = withoutPrefix.substring(0, colon);	2✔
181	const type = Type.fromString(withoutPrefix.substring(colon+1));	2✔
182	next = new TokenWrapper('CONTEXT_REF', { name, type });	2✔
183	} else if (next.startsWith('^^')) {	2,080✔
184	next = new TokenWrapper('ENTITY_TYPE', token.substring('^^'.length));	10✔
185	}
186
187	if (typeof next === 'string')	2,968✔
188	next = new TokenWrapper(next, next);	2,110✔
189	return { done: false, value: next };	2,968✔
190	}
191	}

stanford-oval / thingtalk / 1180

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous