3623061237

Build Type

push

github

Committed by Spencer Phillip Young

Commit Message

gh-40 work with any version of sly

Run Details

4 of 4 new or added lines in 2 files covered. (100.0%)

936 of 953 relevant lines covered (98.22%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.44

/json5/tokenizer.py

import regex as re
import sys
from sly import Lexer
from sly.lex import Token
from json5.utils import JSON5DecodeError
import logging

logger = logging.getLogger(__name__)
# logger.addHandler(logging.StreamHandler(stream=sys.stderr))
# logger.setLevel(level=logging.DEBUG)
class JSON5Token(Token):
    '''
    Representation of a single token.
    '''
    def __init__(self, tok, doc):
        self.type = tok.type
        self.value = tok.value
        self.lineno = tok.lineno
        self.index = tok.index
        self.doc = doc
        self.end = getattr(tok, 'end', None)
    __slots__ = ('type', 'value', 'lineno', 'index', 'doc', 'end')

    def __repr__(self):
        return f'JSON5Token(type={self.type!r}, value={self.value!r}, lineno={self.lineno}, index={self.index}, end={self.end})'


class JSONLexer(Lexer):
    regex_module = re
    reflags = re.DOTALL
    tokens = {LBRACE, RBRACE,
              LBRACKET, RBRACKET,
              DOUBLE_QUOTE_STRING, SINGLE_QUOTE_STRING,
              UNTERMINATED_DOUBLE_QUOTE_STRING,
              UNTERMINATED_SINGLE_QUOTE_STRING,
              NAME,
              COMMA,
              BLOCK_COMMENT,
              LINE_COMMENT,
              WHITESPACE,
              TRUE, FALSE, NULL,
              COLON,

              # Numbers
              PLUS, MINUS,
              FLOAT, INTEGER,
              INFINITY, NAN, EXPONENT,
              HEXADECIMAL,
              OCTAL  # Not allowed, but we capture as a token to raise error later
              }

    def tokenize(self, text, *args, **kwargs):
        for tok in super().tokenize(text, *args, **kwargs):
            tok = JSON5Token(tok, text)
            yield tok

    LBRACE = r'{'
    RBRACE = r'}'
    LBRACKET = r'\['
    RBRACKET = r'\]'
    COLON = r"\:"
    COMMA = r"\,"

    DOUBLE_QUOTE_STRING = r'"(?:[^"\\]|\\.)*"'
    SINGLE_QUOTE_STRING = r"'(?:[^'\\]|\\.)*'"

    LINE_COMMENT = r"//[^\n]*"
    @_(r'/\*((.|\n))*?\*/')
    def BLOCK_COMMENT(self, tok):
        self.lineno += tok.value.count('\n')
        return tok

    @_("[\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u2028\u2029\ufeff]+")
    def WHITESPACE(self, tok):
        self.lineno += tok.value.count('\n')
        return tok

    MINUS = r'\-'
    PLUS = r'\+'
    EXPONENT = r"(e|E)(\-|\+)?\d+"
    HEXADECIMAL = r'0(x|X)[0-9a-fA-F]+'
    OCTAL = r'(0\d+|0o\d+)'
    FLOAT = r'(\d+\.\d*)|(\d*\.\d+)'      # 23.45
    INTEGER = r'\d+'
    NAME = r'[\w_\$\\]([\w_\d\$\\\p{Pc}\p{Mn}\p{Mc}\u200C\u200D])*'

    NAME['true'] = TRUE
    NAME['false'] = FALSE
    NAME['null'] = NULL
    NAME['Infinity'] = INFINITY
    NAME['NaN'] = NAN

    UNTERMINATED_DOUBLE_QUOTE_STRING = r'"(?:[^"\\]|\\.)*'
    UNTERMINATED_SINGLE_QUOTE_STRING = r"'(?:[^'\\]|\\.)*"

    def error(self, t):
        raise JSON5DecodeError(f'Illegal character {t.value[0]!r} at index {self.index}', None)

def tokenize(text):
    lexer = JSONLexer()
    tokens = lexer.tokenize(text)
    return tokens

1	import regex as re	1✔
2	import sys	1✔
3	from sly import Lexer	1✔
4	from sly.lex import Token	1✔
5	from json5.utils import JSON5DecodeError	1✔
6	import logging	1✔
7
8	logger = logging.getLogger(__name__)	1✔
9	# logger.addHandler(logging.StreamHandler(stream=sys.stderr))
10	# logger.setLevel(level=logging.DEBUG)
11	class JSON5Token(Token):	1✔
12	'''
13	Representation of a single token.
14	'''
15	def __init__(self, tok, doc):	1✔
16	self.type = tok.type	1✔
17	self.value = tok.value	1✔
18	self.lineno = tok.lineno	1✔
19	self.index = tok.index	1✔
20	self.doc = doc	1✔
21	self.end = getattr(tok, 'end', None)	1✔
22	__slots__ = ('type', 'value', 'lineno', 'index', 'doc', 'end')	1✔
23
24	def __repr__(self):	1✔
25	return f'JSON5Token(type={self.type!r}, value={self.value!r}, lineno={self.lineno}, index={self.index}, end={self.end})'	×
26
27
28	class JSONLexer(Lexer):	1✔
29	regex_module = re	1✔
30	reflags = re.DOTALL	1✔
31	tokens = {LBRACE, RBRACE,	1✔
32	LBRACKET, RBRACKET,
33	DOUBLE_QUOTE_STRING, SINGLE_QUOTE_STRING,
34	UNTERMINATED_DOUBLE_QUOTE_STRING,
35	UNTERMINATED_SINGLE_QUOTE_STRING,
36	NAME,
37	COMMA,
38	BLOCK_COMMENT,
39	LINE_COMMENT,
40	WHITESPACE,
41	TRUE, FALSE, NULL,
42	COLON,
43
44	# Numbers
45	PLUS, MINUS,
46	FLOAT, INTEGER,
47	INFINITY, NAN, EXPONENT,
48	HEXADECIMAL,
49	OCTAL # Not allowed, but we capture as a token to raise error later
50	}
51
52	def tokenize(self, text, args, *kwargs):	1✔
53	for tok in super().tokenize(text, args, *kwargs):	1✔
54	tok = JSON5Token(tok, text)	1✔
55	yield tok	1✔
56
57	LBRACE = r'{'	1✔
58	RBRACE = r'}'	1✔
59	LBRACKET = r'\['	1✔
60	RBRACKET = r'\]'	1✔
61	COLON = r"\:"	1✔
62	COMMA = r"\,"	1✔
63
64	DOUBLE_QUOTE_STRING = r'"(?:[^"\\]\|\\.)*"'	1✔
65	SINGLE_QUOTE_STRING = r"'(?:[^'\\]\|\\.)*'"	1✔
66
67	LINE_COMMENT = r"//[^\n]*"	1✔
68	@_(r'/\((.\|\n))?\*/')	1✔
69	def BLOCK_COMMENT(self, tok):	1✔
70	self.lineno += tok.value.count('\n')	1✔
71	return tok	1✔
72
73	@_("[\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u2028\u2029\ufeff]+")	1✔
74	def WHITESPACE(self, tok):	1✔
75	self.lineno += tok.value.count('\n')	1✔
76	return tok	1✔
77
78	MINUS = r'\-'	1✔
79	PLUS = r'\+'	1✔
80	EXPONENT = r"(e\|E)(\-\|\+)?\d+"	1✔
81	HEXADECIMAL = r'0(x\|X)[0-9a-fA-F]+'	1✔
82	OCTAL = r'(0\d+\|0o\d+)'	1✔
83	FLOAT = r'(\d+\.\d)\|(\d\.\d+)' # 23.45	1✔
84	INTEGER = r'\d+'	1✔
85	NAME = r'[\w_\$\\]([\w_\d\$\\\p{Pc}\p{Mn}\p{Mc}\u200C\u200D])*'	1✔
86
87	NAME['true'] = TRUE	1✔
88	NAME['false'] = FALSE	1✔
89	NAME['null'] = NULL	1✔
90	NAME['Infinity'] = INFINITY	1✔
91	NAME['NaN'] = NAN	1✔
92
93	UNTERMINATED_DOUBLE_QUOTE_STRING = r'"(?:[^"\\]\|\\.)*'	1✔
94	UNTERMINATED_SINGLE_QUOTE_STRING = r"'(?:[^'\\]\|\\.)*"	1✔
95
96	def error(self, t):	1✔
97	raise JSON5DecodeError(f'Illegal character {t.value[0]!r} at index {self.index}', None)	1✔
98
99	def tokenize(text):	1✔
100	lexer = JSONLexer()	1✔
101	tokens = lexer.tokenize(text)	1✔
102	return tokens	1✔

spyoungtech / json-five / 3623061237

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous