11006888653

Committed 24 Sep 2024 04:16AM UTC coverage: 100.0%. Remained the same

Build # 11006888653

Build Type

Pull #370

github

Committed by

web-flow

Commit Message

Merge f72908f4f into 7acf6578d

Pull Request Pull Request #370: build(deps-dev): bump rollup from 4.21.0 to 4.22.4

Run Details

258 of 258 branches covered (100.0%)

513 of 513 relevant lines covered (100.0%)

287.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/src/function-tokenizer.ts

/**
 * Token type.
 */
export type TokenType =
  | 'ident'
  | '('
  | ')'
  | ','
  | '='
  | '*'
  | 'function'
  | 'class'
  | 'EOF'

/**
 * Lexer Token.
 */
export interface Token {
  type: TokenType
  value?: string
}

/**
 * Flags that can be passed to the tokenizer to toggle certain things.
 */
export const enum TokenizerFlags {
  None = 0,
  /**
   * If this is set, the tokenizer will not attempt to be smart about skipping expressions.
   */
  Dumb = 1,
}

/**
 * Creates a tokenizer for the specified source.
 *
 * @param source
 */
export function createTokenizer(source: string) {
  const end = source.length
  let pos: number = 0
  let type: TokenType = 'EOF'
  let value: string = ''
  let flags = TokenizerFlags.None
  // These are used to greedily skip as much as possible.
  // Whenever we reach a paren, we increment these.
  let parenLeft = 0
  let parenRight = 0

  return {
    next,
    done,
  }

  /**
   * Advances the tokenizer and returns the next token.
   */
  function next(nextFlags = TokenizerFlags.None): Token {
    flags = nextFlags
    advance()
    return createToken()
  }

  /**
   * Advances the tokenizer state.
   */
  function advance() {
    value = ''
    type = 'EOF'

    while (true) {
      if (pos >= end) {
        return (type = 'EOF')
      }

      const ch = source.charAt(pos)
      // Whitespace is irrelevant
      if (isWhiteSpace(ch)) {
        pos++
        continue
      }

      switch (ch) {
        case '(':
          pos++
          parenLeft++
          return (type = ch)
        case ')':
          pos++
          parenRight++
          return (type = ch)
        case '*':
          pos++
          return (type = ch)
        case ',':
          pos++
          return (type = ch)
        case '=':
          pos++
          if ((flags & TokenizerFlags.Dumb) === 0) {
            // Not in dumb-mode, so attempt to skip.
            skipExpression()
          }
          // We need to know that there's a default value so we can
          // skip it if it does not exist when resolving.
          return (type = ch)
        case '/': {
          pos++
          const nextCh = source.charAt(pos)
          if (nextCh === '/') {
            skipUntil((c) => c === '\n', true)
            pos++
          }
          if (nextCh === '*') {
            skipUntil((c) => {
              const closing = source.charAt(pos + 1)
              return c === '*' && closing === '/'
            }, true)
            pos++
          }
          continue
        }
        default:
          // Scans an identifier.
          if (isIdentifierStart(ch)) {
            scanIdentifier()
            return type
          }

          // Elegantly skip over tokens we don't care about.
          pos++
      }
    }
  }

  /**
   * Scans an identifier, given it's already been proven
   * we are ready to do so.
   */
  function scanIdentifier() {
    const identStart = source.charAt(pos)
    const start = ++pos
    while (isIdentifierPart(source.charAt(pos))) {
      pos++
    }
    value = '' + identStart + source.substring(start, pos)
    type = value === 'function' || value === 'class' ? value : 'ident'
    if (type !== 'ident') {
      value = ''
    }
    return value
  }

  /**
   * Skips everything until the next comma or the end of the parameter list.
   * Checks the parenthesis balance so we correctly skip function calls.
   */
  function skipExpression() {
    skipUntil((ch) => {
      const isAtRoot = parenLeft === parenRight + 1
      if (ch === ',' && isAtRoot) {
        return true
      }

      if (ch === '(') {
        parenLeft++
        return false
      }

      if (ch === ')') {
        parenRight++
        if (isAtRoot) {
          return true
        }
      }

      return false
    })
  }

  /**
   * Skips strings and whilespace until the predicate is true.
   *
   * @param callback stops skipping when this returns `true`.
   * @param dumb if `true`, does not skip whitespace and strings;
   * it only stops once the callback returns `true`.
   */
  function skipUntil(callback: (ch: string) => boolean, dumb = false) {
    while (pos < source.length) {
      const ch = source.charAt(pos)
      if (callback(ch)) {
        return
      }

      if (!dumb) {
        if (isWhiteSpace(ch)) {
          pos++
          continue
        }

        if (isStringQuote(ch)) {
          skipString()
          continue
        }
      }
      pos++
    }
  }

  /**
   * Given the current position is at a string quote, skips the entire string.
   */
  function skipString() {
    const quote = source.charAt(pos)
    pos++
    while (pos < source.length) {
      const ch = source.charAt(pos)
      const prev = source.charAt(pos - 1)
      // Checks if the quote was escaped.
      if (ch === quote && prev !== '\\') {
        pos++
        return
      }

      // Template strings are a bit tougher, we want to skip the interpolated values.
      if (quote === '`') {
        const next = source.charAt(pos + 1)
        if (next === '$') {
          const afterDollar = source.charAt(pos + 2)
          if (afterDollar === '{') {
            // This is the start of an interpolation; skip the ${
            pos = pos + 2
            // Skip strings and whitespace until we reach the ending }.
            // This includes skipping nested interpolated strings. :D
            skipUntil((ch) => ch === '}')
          }
        }
      }

      pos++
    }
  }

  /**
   * Creates a token from the current state.
   */
  function createToken(): Token {
    if (value) {
      return { value, type }
    }
    return { type }
  }

  /**
   * Determines if we are done parsing.
   */
  function done() {
    return type === 'EOF'
  }
}

/**
 * Determines if the given character is a whitespace character.
 *
 * @param  {string}  ch
 * @return {boolean}
 */
function isWhiteSpace(ch: string): boolean {
  switch (ch) {
    case '\r':
    case '\n':
    case ' ':
      return true
  }
  return false
}

/**
 * Determines if the specified character is a string quote.
 * @param  {string}  ch
 * @return {boolean}
 */
function isStringQuote(ch: string): boolean {
  switch (ch) {
    case "'":
    case '"':
    case '`':
      return true
  }
  return false
}

// NOTE: I've added the `.` character so that member expression paths
// are seen as identifiers. This is so we don't get a constructor token for
// stuff like `MyClass.prototype.constructor()`
const IDENT_START_EXPR = /^[_$a-zA-Z\xA0-\uFFFF]$/
const IDENT_PART_EXPR = /^[._$a-zA-Z0-9\xA0-\uFFFF]$/

/**
 * Determines if the character is a valid JS identifier start character.
 */
function isIdentifierStart(ch: string) {
  return IDENT_START_EXPR.test(ch)
}

/**
 * Determines if the character is a valid JS identifier start character.
 */
function isIdentifierPart(ch: string) {
  return IDENT_PART_EXPR.test(ch)
}

1	/**
2	* Token type.
3	*/
4	export type TokenType =
5	\| 'ident'
6	\| '('
7	\| ')'
8	\| ','
9	\| '='
10	\| '*'
11	\| 'function'
12	\| 'class'
13	\| 'EOF'
14
15	/**
16	* Lexer Token.
17	*/
18	export interface Token {
19	type: TokenType
20	value?: string
21	}
22
23	/**
24	* Flags that can be passed to the tokenizer to toggle certain things.
25	*/
26	export const enum TokenizerFlags {
27	None = 0,
28	/**
29	* If this is set, the tokenizer will not attempt to be smart about skipping expressions.
30	*/
31	Dumb = 1,
32	}
33
34	/**
35	* Creates a tokenizer for the specified source.
36	*
37	* @param source
38	*/
39	export function createTokenizer(source: string) {	26✔
40	const end = source.length	478✔
41	let pos: number = 0	478✔
42	let type: TokenType = 'EOF'	478✔
43	let value: string = ''	478✔
44	let flags = TokenizerFlags.None	478✔
45	// These are used to greedily skip as much as possible.
46	// Whenever we reach a paren, we increment these.
47	let parenLeft = 0	478✔
48	let parenRight = 0	478✔
49
50	return {	478✔
51	next,
52	done,
53	}
54
55	/**
56	* Advances the tokenizer and returns the next token.
57	*/
58	function next(nextFlags = TokenizerFlags.None): Token {	199✔
59	flags = nextFlags	2,112✔
60	advance()	2,112✔
61	return createToken()	2,112✔
62	}
63
64	/**
65	* Advances the tokenizer state.
66	*/
67	function advance() {
68	value = ''	2,112✔
69	type = 'EOF'	2,112✔
70
71	while (true) {	2,112✔
72	if (pos >= end) {	5,354✔
73	return (type = 'EOF')	82✔
74	}
75
76	const ch = source.charAt(pos)	5,272✔
77	// Whitespace is irrelevant
78	if (isWhiteSpace(ch)) {	5,272✔
79	pos++	2,710✔
80	continue	2,710✔
81	}
82
83	switch (ch) {	2,562✔
84	case '(':
85	pos++	426✔
86	parenLeft++	426✔
87	return (type = ch)	426✔
88	case ')':
89	pos++	378✔
90	parenRight++	378✔
91	return (type = ch)	378✔
92	case '*':
93	pos++	4✔
94	return (type = ch)	4✔
95	case ',':
96	pos++	106✔
97	return (type = ch)	106✔
98	case '=':
99	pos++	80✔
100	if ((flags & TokenizerFlags.Dumb) === 0) {	80✔
101	// Not in dumb-mode, so attempt to skip.
102	skipExpression()	68✔
103	}
104	// We need to know that there's a default value so we can
105	// skip it if it does not exist when resolving.
106	return (type = ch)	80✔
107	case '/': {
108	pos++	66✔
109	const nextCh = source.charAt(pos)	66✔
110	if (nextCh === '/') {	66✔
111	skipUntil((c) => c === '\n', true)	92✔
112	pos++	6✔
113	}
114	if (nextCh === '*') {	66✔
115	skipUntil((c) => {	30✔
116	const closing = source.charAt(pos + 1)	708✔
117	return c === '*' && closing === '/'	708✔
118	}, true)
119	pos++	30✔
120	}
121	continue	66✔
122	}
123	default:
124	// Scans an identifier.
125	if (isIdentifierStart(ch)) {	1,502✔
126	scanIdentifier()	1,036✔
127	return type	1,036✔
128	}
129
130	// Elegantly skip over tokens we don't care about.
131	pos++	466✔
132	}
133	}
134	}
135
136	/**
137	* Scans an identifier, given it's already been proven
138	* we are ready to do so.
139	*/
140	function scanIdentifier() {
141	const identStart = source.charAt(pos)	1,036✔
142	const start = ++pos	1,036✔
143	while (isIdentifierPart(source.charAt(pos))) {	1,036✔
144	pos++	5,944✔
145	}
146	value = '' + identStart + source.substring(start, pos)	1,036✔
147	type = value === 'function' \|\| value === 'class' ? value : 'ident'	1,036✔
148	if (type !== 'ident') {	1,036✔
149	value = ''	310✔
150	}
151	return value	1,036✔
152	}
153
154	/**
155	* Skips everything until the next comma or the end of the parameter list.
156	* Checks the parenthesis balance so we correctly skip function calls.
157	*/
158	function skipExpression() {
159	skipUntil((ch) => {	68✔
160	const isAtRoot = parenLeft === parenRight + 1	452✔
161	if (ch === ',' && isAtRoot) {	452✔
162	return true	34✔
163	}
164
165	if (ch === '(') {	418✔
166	parenLeft++	14✔
167	return false	14✔
168	}
169
170	if (ch === ')') {	404✔
171	parenRight++	42✔
172	if (isAtRoot) {	42✔
173	return true	30✔
174	}
175	}
176
177	return false	374✔
178	})
179	}
180
181	/**
182	* Skips strings and whilespace until the predicate is true.
183	*
184	* @param callback stops skipping when this returns `true`.
185	* @param dumb if `true`, does not skip whitespace and strings;
186	* it only stops once the callback returns `true`.
187	*/
188	function skipUntil(callback: (ch: string) => boolean, dumb = false) {	39✔
189	while (pos < source.length) {	114✔
190	const ch = source.charAt(pos)	1,382✔
191	if (callback(ch)) {	1,382✔
192	return	110✔
193	}
194
195	if (!dumb) {	1,272✔
196	if (isWhiteSpace(ch)) {	508✔
197	pos++	122✔
198	continue	122✔
199	}
200
201	if (isStringQuote(ch)) {	386✔
202	skipString()	48✔
203	continue	48✔
204	}
205	}
206	pos++	1,102✔
207	}
208	}
209
210	/**
211	* Given the current position is at a string quote, skips the entire string.
212	*/
213	function skipString() {
214	const quote = source.charAt(pos)	48✔
215	pos++	48✔
216	while (pos < source.length) {	48✔
217	const ch = source.charAt(pos)	530✔
218	const prev = source.charAt(pos - 1)	530✔
219	// Checks if the quote was escaped.
220	if (ch === quote && prev !== '\\') {	530✔
221	pos++	48✔
222	return	48✔
223	}
224
225	// Template strings are a bit tougher, we want to skip the interpolated values.
226	if (quote === '`') {	482✔
227	const next = source.charAt(pos + 1)	198✔
228	if (next === '$') {	198✔
229	const afterDollar = source.charAt(pos + 2)	12✔
230	if (afterDollar === '{') {	12✔
231	// This is the start of an interpolation; skip the ${
232	pos = pos + 2	10✔
233	// Skip strings and whitespace until we reach the ending }.
234	// This includes skipping nested interpolated strings. :D
235	skipUntil((ch) => ch === '}')	130✔
236	}
237	}
238	}
239
240	pos++	482✔
241	}
242	}
243
244	/**
245	* Creates a token from the current state.
246	*/
247	function createToken(): Token {
248	if (value) {	2,112✔
249	return { value, type }	726✔
250	}
251	return { type }	1,386✔
252	}
253
254	/**
255	* Determines if we are done parsing.
256	*/
257	function done() {
258	return type === 'EOF'	2,056✔
259	}
260	}
261
262	/**
263	* Determines if the given character is a whitespace character.
264	*
265	* @param {string} ch
266	* @return {boolean}
267	*/
268	function isWhiteSpace(ch: string): boolean {
269	switch (ch) {	5,780✔
270	case '\r':
271	case '\n':
272	case ' ':
273	return true	2,832✔
274	}
275	return false	2,948✔
276	}
277
278	/**
279	* Determines if the specified character is a string quote.
280	* @param {string} ch
281	* @return {boolean}
282	*/
283	function isStringQuote(ch: string): boolean {
284	switch (ch) {	386✔
285	case "'":
286	case '"':
287	case '`':
288	return true	48✔
289	}
290	return false	338✔
291	}
292
293	// NOTE: I've added the `.` character so that member expression paths
294	// are seen as identifiers. This is so we don't get a constructor token for
295	// stuff like `MyClass.prototype.constructor()`
296	const IDENT_START_EXPR = /^[_$a-zA-Z\xA0-\uFFFF]$/	26✔
297	const IDENT_PART_EXPR = /^[._$a-zA-Z0-9\xA0-\uFFFF]$/	26✔
298
299	/**
300	* Determines if the character is a valid JS identifier start character.
301	*/
302	function isIdentifierStart(ch: string) {
303	return IDENT_START_EXPR.test(ch)	1,502✔
304	}
305
306	/**
307	* Determines if the character is a valid JS identifier start character.
308	*/
309	function isIdentifierPart(ch: string) {
310	return IDENT_PART_EXPR.test(ch)	6,980✔
311	}

jeffijoe / awilix / 11006888653

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous