#137

Committed 04 May 2018 03:54AM UTC coverage: 94.822% (-0.9%) from 95.69%

Build # #137

Build Type

push

travis-ci

Committed by

mblarsen

Commit Message

change: Make casting of types work like syntax RX_PARAM

It was a problem that you could not have a string value of say "true" if
you wanted to. So numbers and booleans are not longer matched when using
".

Coverage Stats

117 of 126 branches covered (92.86%)

Branch coverage included in aggregate %.

5 of 5 new or added lines in 1 file covered. (100.0%)

7 existing lines in 1 file now uncovered.

176 of 183 relevant lines covered (96.17%)

31.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.82

/src/shortcode-tokenizer.js

/** @module ShortcodeTokenizer */

/* tokens */
const TEXT = 'TEXT'
const ERROR = 'ERROR'
const OPEN = 'OPEN'
const CLOSE = 'CLOSE'
const SELF_CLOSING = 'SELF_CLOSING'

/* eslint-disable */

/* matches code name */
const RX_KEY = '[a-zA-Z][a-zA-Z0-9_-]*'

/* matches paramters */
const RX_PARAM =       RX_KEY + '=\\d+\\.\\d+' +         // floats
                 '|' + RX_KEY + '=\\d+' +                // ints
                 '|' + RX_KEY + '=(true|false|yes|no)' + // bools
                 '|' + RX_KEY + '="[^\\]"]*"' +          // double-qouted strings
                 '|' + RX_KEY + '=\'[^\\]\']*\'' +       // single-qouted strings
                 '|' + RX_KEY                            // flags
const RX_PARAMS = '(?:(?:' + RX_PARAM + ')(?:(?!\\s+/?\\])\\s|))+'

/* matches all code token types, used for quickly
   finding potentia code tokens */
const RX_ENCLOSURE   = '\\[\\/?[a-zA-Z][^\\]]+\\]'
/* matches opening code tokens [row] */
const RX_OPEN        = '\\[(' + RX_KEY + ')(\\s' + RX_PARAMS + ')?\\]'
/* matches self-closing code tokens [row/] */
const RX_SELFCLOSING = '\\[(' + RX_KEY + ')(\\s' + RX_PARAMS + ')?\\s?\\/\\]'
/* matches close code tokens [/row] */
const RX_CLOSE       = '\\[\\/(' + RX_KEY + ')\\]'

/* case-insensitive regular expressions */
const rxParams      = new RegExp(RX_PARAMS.substring(0, RX_PARAMS.length - 1), 'ig')
const rxEnclosure   = new RegExp(RX_ENCLOSURE, 'i')
const rxOpen        = new RegExp(RX_OPEN, 'i')
const rxClose       = new RegExp(RX_CLOSE, 'i')
const rxSelfclosing = new RegExp(RX_SELFCLOSING, 'i')

/* eslint-enable */

/**
 * Get token type based on token-string.
 *
 * Note: assuming that this is not a TEXT token
 *
 * @param {string} str
 * @returns {string} token type
 */
function getTokenType(str) {
  if (str[1] === '/') {
    return CLOSE
  }
  if (str[str.length - 2] === '/') {
    return SELF_CLOSING
  }
  return OPEN
}

/**
 * Casts input string to native types.
 *
 * @param {string} value
 * @returns {*} mixed value
 */
function castValue(value) {
  if (/^\d+$/.test(value)) return Number(value)
  if (/^\d+.\d+$/.test(value)) return Number(value)
  if (/^(true|false|yes|no)$/i.test(value)) {
    value = value.toLowerCase()
    return value === 'true' || value === 'yes'
  }
  return value.replace(/(^['"]|['"]$)/g, '')
}

/**
 * Token class is used both as a token during tokenization/lexing
 * and as a node in the resulting AST.
 *
 * @access private
 */
export class Token {
  constructor(type, body, pos = 0) {
    this.name = null
    this.type = type
    this.body = body
    this.pos = pos
    this.children = []
    this.params = {}
    this.isClosed = type === SELF_CLOSING
    this.init()
  }

  /**
   * @access private
   */
  init() {
    if (this.type !== TEXT && this.type !== ERROR) {
      const match = this.matchBody()
      this.initName(match)
      if (match[2]) {
        this.initParams(match[2])
      }
    }
  }

  /**
   * @access private
   */
  initName(match) {
    this.name = match[1]
  }

  /**
   * @access private
   */
  initParams(paramStr) {
    const match = paramStr.match(rxParams)
    this.params = match.reduce((params, paramToken) => {
      paramToken = paramToken.trim()
      let equal = paramToken.indexOf('=')
      if (!~equal) {
        params[paramToken] = true
      } else {
        params[paramToken.substring(0, equal)] = castValue(paramToken.substring(equal + 1))
      }
      return params
    }, {})
  }

  /**
   * @access private
   */
  matchBody() {
    let rx
    if (this.type === CLOSE) {
      rx = rxClose
    } else if (this.type === OPEN) {
      rx = rxOpen
    } else if (this.type === SELF_CLOSING) {
      rx = rxSelfclosing
    } else {
      throw new SyntaxError('Unknown token: ' + this.type)
    }

    let match = this.body.match(rx)
    if (match === null) {
      throw new SyntaxError('Invalid ' + this.type + ' token: ' + this.body)
    }
    return match
  }

  /**
   * Determines if this token can close the param token.
   *
   * @access public
   * @param {Token} token another token
   * @returns {boolean}
   */
  canClose(token) {
    return this.name === token.name
  }
}

/**
 * Creates a new tokenizer.
 *
 * Pass in input as first param or later using `input()`
 *
 * @param {string} [input=null] Optional input to tokenize
 * @param {Object} [options] options object
 * @param {boolean} [options.strict=true] strict mode
 * @param {boolean} [options.skipWhiteSpace=false] will ignore tokens containing only white space (basically all \s)
 */
export default class ShortcodeTokenizer {

  constructor(input = null, options = {strict: true, skipWhiteSpace: false}) {
    if (typeof options === 'boolean') {
      options = {strict: options, skipWhiteSpace: false}
    }
    this.options = Object.assign({strict: true, skipWhiteSpace: false}, options)
    this.buf = null
    this.originalBuf = null
    this.pos = 0
    if (input) {
      this.input(input)
    }
  }

  /**
   * @deprecated use options.strict
   */
  get strict() {
    console.warn(`Deprecated: use options.strict instead`)
    return this.options.strict
  }

  /**
   * @deprecated use options.strict
   */
  set strict(value) {
    console.warn(`Deprecated: use options.strict = ${value} instead`)
    this.options.strict = value
  }

  /**
   * Sets input buffer with a new input string.
   *
   * @param {string} input template string
   * @throws {Error} Invalid input
   * @returns {this} returns this for chaining
   */
  input(input) {
    if (typeof input !== 'string') {
      throw new Error('Invalid input')
    }

    this.buf = this.originalBuf = input
    this.pos = 0
    return this
  }

  /**
   * Resets input buffer and position to their origial values.
   *
   * @returns {this} returns this for chaining
   */
  reset() {
    this.buf = this.originalBuf
    this.pos = 0
    return this
  }

  /**
   * Creates a token generator.
   *
   * @throws {Error} Invalid input
   * @returns {Token[]} An array of Token instances
   */
  tokens(input = null) {
    if (input) {
      this.input(input)
    }

    if (typeof this.buf !== 'string') {
      throw new Error('Invalid input')
    }

    let tokens = []
    let allTokens = []
    while ((tokens = this._next()) !== null) {
      tokens = Array.isArray(tokens) ? tokens : [tokens]
      allTokens.push(...tokens)
    }
    return allTokens
  }

  /**
   * Uses the tokens generator to build an AST from the tokens.
   *
   * @see tokens
   * @returns {array} an array of AST roots
   */
  ast(input = null) {
    let tokens = this.tokens(input)
    let stack = []
    let ast = []
    let parent = null
    let token
    for (token of tokens) {
      if (token.type === TEXT) {
        if (this.options.skipWhiteSpace && token.body.replace(/\s+/g, '').length === 0) {
          continue
        }
        if (!parent) {
          ast.push(token)
        } else {
          parent.children.push(token)
        }
      } else if (token.type === OPEN) {
        if (!parent) {
          parent = token
          ast.push(parent)
        } else {
          parent.children.push(token)
          stack.push(parent)
          parent = token
        }
      } else if (token.type === CLOSE) {
        if (!parent || !token.canClose(parent)) {
          if (this.options.strict) {
            throw new SyntaxError('Unmatched close token: ' + token.body)
          } else {
            let err = new Token(ERROR, token.body)
            if (!parent) {
              ast.push(err)
            } else {
              parent.children.push(err)
            }
          }
        } else {
          parent.isClosed = true
          parent = stack.pop()
        }
      } else if (token.type === SELF_CLOSING) {
        if (!parent) {
          ast.push(token)
        } else {
          parent.children.push(token)
        }
      } else {
        throw new SyntaxError('Unknown token: ' + token.type)
      }
    }
    if (parent) {
      if (this.options.strict) {
        throw new SyntaxError('Unmatched open token: ' + parent.body)
      } else {
        ast.push(new Token(ERROR, token.body))
      }
    }
    return ast
  }

  /**
   * Internal function used to retrieve the next token from the current
   * position in the input buffer.
   *
   * @private
   * @returns {Token} returns the next Token from the input buffer
   */
  _next() {
    if (!this.buf) {
      return null
    }

    let match = this.buf.match(rxEnclosure)

    // all text
    if (match === null) {
      let token = new Token(TEXT, this.buf, this.pos)
      this.pos += this.buf.length
      this.buf = null
      return token
    }

    let tokens = []

    // first part is text
    if (match.index !== 0) {
      tokens.push(new Token(
        TEXT,
        this.buf.substring(0, match.index),
        this.pos
      ))
    }

    // matching token
    tokens.push(new Token(
      getTokenType(match[0]),
      match[0],
      this.pos + match.index
    ))

    // shorten buffer
    this.buf = this.buf.substring(match.index + match[0].length)
    this.pos += match.index + match[0].length
    if (this.buf.length === 0) {
      this.buf = null
    }
    return tokens
  }
}

Object.assign(ShortcodeTokenizer, {
  TEXT,
  ERROR,
  OPEN,
  CLOSE,
  SELF_CLOSING,
  rxParams,
  rxEnclosure,
  rxOpen,
  rxClose,
  rxSelfclosing
})

1	/** @module ShortcodeTokenizer */
2
3	/* tokens */
4	const TEXT = 'TEXT'	1✔
5	const ERROR = 'ERROR'	1✔
6	const OPEN = 'OPEN'	1✔
7	const CLOSE = 'CLOSE'	1✔
8	const SELF_CLOSING = 'SELF_CLOSING'	1✔
9
10	/* eslint-disable */
11
12	/* matches code name */
13	const RX_KEY = '[a-zA-Z][a-zA-Z0-9_-]*'	1✔
14
15	/* matches paramters */
16	const RX_PARAM = RX_KEY + '=\\d+\\.\\d+' + // floats	1✔
17	'\|' + RX_KEY + '=\\d+' + // ints
18	'\|' + RX_KEY + '=(true\|false\|yes\|no)' + // bools
19	'\|' + RX_KEY + '="[^\\]"]*"' + // double-qouted strings
20	'\|' + RX_KEY + '=\'[^\\]\']*\'' + // single-qouted strings
21	'\|' + RX_KEY // flags
22	const RX_PARAMS = '(?:(?:' + RX_PARAM + ')(?:(?!\\s+/?\\])\\s\|))+'	1✔
23
24	/* matches all code token types, used for quickly
25	finding potentia code tokens */
26	const RX_ENCLOSURE = '\\[\\/?[a-zA-Z][^\\]]+\\]'	1✔
27	/* matches opening code tokens [row] */
28	const RX_OPEN = '\\[(' + RX_KEY + ')(\\s' + RX_PARAMS + ')?\\]'	1✔
29	/* matches self-closing code tokens [row/] */
30	const RX_SELFCLOSING = '\\[(' + RX_KEY + ')(\\s' + RX_PARAMS + ')?\\s?\\/\\]'	1✔
31	/* matches close code tokens [/row] */
32	const RX_CLOSE = '\\[\\/(' + RX_KEY + ')\\]'	1✔
33
34	/* case-insensitive regular expressions */
35	const rxParams = new RegExp(RX_PARAMS.substring(0, RX_PARAMS.length - 1), 'ig')	1✔
36	const rxEnclosure = new RegExp(RX_ENCLOSURE, 'i')	1✔
37	const rxOpen = new RegExp(RX_OPEN, 'i')	1✔
38	const rxClose = new RegExp(RX_CLOSE, 'i')	1✔
39	const rxSelfclosing = new RegExp(RX_SELFCLOSING, 'i')	1✔
40
41	/* eslint-enable */
42
43	/**
44	* Get token type based on token-string.
45	*
46	* Note: assuming that this is not a TEXT token
47	*
48	* @param {string} str
49	* @returns {string} token type
50	*/
51	function getTokenType(str) {
52	if (str[1] === '/') {	75✔
53	return CLOSE	31✔
54	}
55	if (str[str.length - 2] === '/') {	44✔
56	return SELF_CLOSING	13✔
57	}
58	return OPEN	31✔
59	}
60
61	/**
62	* Casts input string to native types.
63	*
64	* @param {string} value
65	* @returns {*} mixed value
66	*/
67	function castValue(value) {
68	if (/^\d+$/.test(value)) return Number(value)	19✔
69	if (/^\d+.\d+$/.test(value)) return Number(value)	14✔
70	if (/^(true\|false\|yes\|no)$/i.test(value)) {	12✔
71	value = value.toLowerCase()	5✔
72	return value === 'true' \|\| value === 'yes'	5✔
73	}
74	return value.replace(/(^['"]\|['"]$)/g, '')	7✔
75	}
76
77	/**
78	* Token class is used both as a token during tokenization/lexing
79	* and as a node in the resulting AST.
80	*
81	* @access private
82	*/
83	export class Token {
84	constructor(type, body, pos = 0) {	33✔
85	this.name = null	164✔
86	this.type = type	164✔
87	this.body = body	164✔
88	this.pos = pos	164✔
89	this.children = []	164✔
90	this.params = {}	164✔
91	this.isClosed = type === SELF_CLOSING	164✔
92	this.init()	164✔
93	}
94
95	/**
96	* @access private
97	*/
98	init() {
99	if (this.type !== TEXT && this.type !== ERROR) {	164✔
100	const match = this.matchBody()	109✔
101	this.initName(match)	104✔
102	if (match[2]) {	104✔
103	this.initParams(match[2])	17✔
104	}
105	}
106	}
107
108	/**
109	* @access private
110	*/
111	initName(match) {
112	this.name = match[1]	104✔
113	}
114
115	/**
116	* @access private
117	*/
118	initParams(paramStr) {
119	const match = paramStr.match(rxParams)	17✔
120	this.params = match.reduce((params, paramToken) => {	17✔
121	paramToken = paramToken.trim()	22✔
122	let equal = paramToken.indexOf('=')	22✔
123	if (!~equal) {	22✔
124	params[paramToken] = true	3✔
125	} else {
126	params[paramToken.substring(0, equal)] = castValue(paramToken.substring(equal + 1))	19✔
127	}
128	return params	22✔
129	}, {})
130	}
131
132	/**
133	* @access private
134	*/
135	matchBody() {
136	let rx
137	if (this.type === CLOSE) {	109✔
138	rx = rxClose	34✔
139	} else if (this.type === OPEN) {	75✔
140	rx = rxOpen	55✔
141	} else if (this.type === SELF_CLOSING) {	20✔
142	rx = rxSelfclosing	18✔
143	} else {
144	throw new SyntaxError('Unknown token: ' + this.type)	2✔
145	}
146
147	let match = this.body.match(rx)	107✔
148	if (match === null) {	107✔
149	throw new SyntaxError('Invalid ' + this.type + ' token: ' + this.body)	3✔
150	}
151	return match	104✔
152	}
153
154	/**
155	* Determines if this token can close the param token.
156	*
157	* @access public
158	* @param {Token} token another token
159	* @returns {boolean}
160	*/
161	canClose(token) {
162	return this.name === token.name	27✔
163	}
164	}
165
166	/**
167	* Creates a new tokenizer.
168	*
169	* Pass in input as first param or later using `input()`
170	*
171	* @param {string} [input=null] Optional input to tokenize
172	* @param {Object} [options] options object	12✔
173	* @param {boolean} [options.strict=true] strict mode	8✔
174	* @param {boolean} [options.skipWhiteSpace=false] will ignore tokens containing only white space (basically all \s)	8✔
175	*/	5✔
176	export default class ShortcodeTokenizer {	1!
177		1✔
178	constructor(input = null, options = {strict: true, skipWhiteSpace: false}) {
179	if (typeof options === 'boolean') {
180	options = {strict: options, skipWhiteSpace: false}
181	}
182	this.options = Object.assign({strict: true, skipWhiteSpace: false}, options)
183	this.buf = null
184	this.originalBuf = null
185	this.pos = 0
186	if (input) {
187	this.input(input)
188	}	6✔
189	}	15✔
190		1✔
191	/**
192	* @deprecated use options.strict
193	*/	14✔
194	get strict() {	2✔
195	console.warn(`Deprecated: use options.strict instead`)
196	return this.options.strict
197	}
198		12✔
199	/**	9✔
200	* @deprecated use options.strict	3✔
201	*/	1✔
202	set strict(value) {
203	console.warn(`Deprecated: use options.strict = ${value} instead`)	2✔
204	this.options.strict = value	4✔
205	}	4!
206		4✔
207	/**	2!
208	* Sets input buffer with a new input string.	2✔
209	*
210	* @param {string} input template string
211	* @throws {Error} Invalid input
212	* @returns {this} returns this for chaining	12✔
213	*/
214	input(input) {	12✔
215	if (typeof input !== 'string') {	10✔
216	throw new Error('Invalid input')	2!
217	}	2✔
218
219	this.buf = this.originalBuf = input
220	this.pos = 0
221	return this
222	}
223
224	/**
225	* Resets input buffer and position to their origial values.
226	*
227	* @returns {this} returns this for chaining
228	*/
229	reset() {
230	this.buf = this.originalBuf
231	this.pos = 0
232	return this
233	}
234		90✔
235	/**	50✔
236	* Creates a token generator.	2✔
237	*
238	* @throws {Error} Invalid input	50✔
239	* @returns {Token[]} An array of Token instances	50✔
240	*/	50✔
241	tokens(input = null) {	50✔
242	if (input) {	50✔
243	this.input(input)	1✔
244	}
245
246	if (typeof this.buf !== 'string') {
247	throw new Error('Invalid input')
248	}
249
250	let tokens = []
UNCOV 251	let allTokens = []	×
UNCOV 252	while ((tokens = this._next()) !== null) {	×
253	tokens = Array.isArray(tokens) ? tokens : [tokens]
254	allTokens.push(...tokens)
255	}
256	return allTokens
257	}
258
UNCOV 259	/**	×
UNCOV 260	* Uses the tokens generator to build an AST from the tokens.	×
261	*
262	* @see tokens
263	* @returns {array} an array of AST roots
264	*/
265	ast(input = null) {
266	let tokens = this.tokens(input)
267	let stack = []
268	let ast = []
269	let parent = null
270	let token
271	for (token of tokens) {	52✔
272	if (token.type === TEXT) {	2✔
273	if (this.options.skipWhiteSpace && token.body.replace(/\s+/g, '').length === 0) {
274	continue
275	}	50✔
276	if (!parent) {	50✔
277	ast.push(token)	50✔
278	} else {
279	parent.children.push(token)
280	}
281	} else if (token.type === OPEN) {
282	if (!parent) {
283	parent = token
284	ast.push(parent)
285	} else {
286	parent.children.push(token)	1✔
287	stack.push(parent)	1✔
288	parent = token	1✔
289	}
290	} else if (token.type === CLOSE) {
291	if (!parent \|\| !token.canClose(parent)) {
292	if (this.options.strict) {
293	throw new SyntaxError('Unmatched close token: ' + token.body)
294	} else {
295	let err = new Token(ERROR, token.body)
296	if (!parent) {
297	ast.push(err)	27✔
298	} else {	52✔
299	parent.children.push(err)	9✔
300	}
301	}
302	} else {	50✔
303	parent.isClosed = true	1✔
304	parent = stack.pop()
305	}
306	} else if (token.type === SELF_CLOSING) {	49✔
307	if (!parent) {	49✔
308	ast.push(token)	49✔
309	} else {	89✔
310	parent.children.push(token)	89✔
311	}
312	} else {	49✔
313	throw new SyntaxError('Unknown token: ' + token.type)
314	}
315	}
316	if (parent) {
317	if (this.options.strict) {
318	throw new SyntaxError('Unmatched open token: ' + parent.body)
319	} else {
320	ast.push(new Token(ERROR, token.body))
321	}	16✔
322	}	21✔
323	return ast	21✔
324	}	21✔
325		21✔
326	/**
327	* Internal function used to retrieve the next token from the current	21✔
328	* position in the input buffer.	78✔
329	*	17✔
330	* @private	2✔
331	* @returns {Token} returns the next Token from the input buffer
332	*/	15✔
333	_next() {	2✔
334	if (!this.buf) {
335	return null	13✔
336	}
337		61✔
338	let match = this.buf.match(rxEnclosure)	28✔
339		17✔
340	// all text	17✔
341	if (match === null) {
342	let token = new Token(TEXT, this.buf, this.pos)	11✔
343	this.pos += this.buf.length	11✔
344	this.buf = null	11✔
345	return token
346	}	33✔
347		29✔
348	let tokens = []	4✔
349		3✔
350	// first part is text
351	if (match.index !== 0) {	1✔
352	tokens.push(new Token(	1!
353	TEXT,	1✔
354	this.buf.substring(0, match.index),
UNCOV 355	this.pos	×
356	))
357	}
358
359	// matching token	25✔
360	tokens.push(new Token(	25✔
361	getTokenType(match[0]),
362	match[0],	4!
363	this.pos + match.index	4✔
364	))	2✔
365
366	// shorten buffer	2✔
367	this.buf = this.buf.substring(match.index + match[0].length)
368	this.pos += match.index + match[0].length
UNCOV 369	if (this.buf.length === 0) {	×
370	this.buf = null
371	}
372	return tokens	18✔
373	}	1!
374	}	1✔
375
UNCOV 376	Object.assign(ShortcodeTokenizer, {	×
377	TEXT,
378	ERROR,
379	OPEN,	17✔
380	CLOSE,
381	SELF_CLOSING,
382	rxParams,
383	rxEnclosure,
384	rxOpen,
385	rxClose,
386	rxSelfclosing
387	})

mblarsen / shortcode-tokenizer / #137

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous