10749342211

Committed 05 Sep 2024 11:09PM UTC coverage: 98.871% (-1.1%) from 100.0%

Build # 10749342211

Build Type

push

github

Committed by

web-flow

Commit Message

Refactor lexical analysis (#390)

* return begin/end iterators as token contents instead of unconditionally copying them into a work buffer

* scan scalar contents in deserializer, not in lexer

* reduce string copies when scanning scalars

* improved block scalar scanning

* separate plain scalar scannings based on inside/outside flow contexts

* made all lines/branches covered in the unit test app

* fixed build errors on macos / windows

Run Details

1171 of 1206 branches covered (97.1%)

Branch coverage included in aggregate %.

246 of 249 new or added lines in 6 files covered. (98.8%)

2 existing lines in 2 files now uncovered.

4259 of 4286 relevant lines covered (99.37%)

331.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.84

/include/fkYAML/detail/input/scalar_scanner.hpp

///  _______   __ __   __  _____   __  __  __
/// |   __| |_/  |  \_/  |/  _  \ /  \/  \|  |     fkYAML: A C++ header-only YAML library
/// |   __|  _  < \_   _/|  ___  |    _   |  |___  version 0.3.11
/// |__|  |_| \__|  |_|  |_|   |_|___||___|______| https://github.com/fktn-k/fkYAML
///
/// SPDX-FileCopyrightText: 2023-2024 Kensuke Fukutani <fktn.dev@gmail.com>
/// SPDX-License-Identifier: MIT
///
/// @file

#ifndef FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_
#define FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_

#include <algorithm>
#include <string>

#include <fkYAML/detail/macros/version_macros.hpp>
#include <fkYAML/detail/assert.hpp>
#include <fkYAML/node_type.hpp>

FK_YAML_DETAIL_NAMESPACE_BEGIN

namespace {

/// @brief Check if the given character is a digit.
/// @note This function is needed to avoid assertion failures in `std::isdigit()` especially when compiled with MSVC.
/// @param c A character to be checked.
/// @return true if the given character is a digit, false otherwise.
inline bool is_digit(char c) {
    return ('0' <= c && c <= '9');
}

/// @brief Check if the given character is a hex-digit.
/// @note This function is needed to avoid assertion failures in `std::isxdigit()` especially when compiled with MSVC.
/// @param c A character to be checked.
/// @return true if the given character is a hex-digit, false otherwise.
inline bool is_xdigit(char c) {
    return (('0' <= c && c <= '9') || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f'));
}

} // namespace
class scalar_scanner {
public:
    static node_type scan(const std::string& token) {
        switch (token.size()) {
        case 0:
            return node_type::STRING;
        case 1:
            if (token[0] == '~') {
                return node_type::NULL_OBJECT;
            }
            break;
        case 4:
            switch (token[0]) {
            case 'n':
            case 'N':
                if (token == "null" || token == "Null" || token == "NULL") {
                    return node_type::NULL_OBJECT;
                }
                break;
            case 't':
            case 'T':
                if (token == "true" || token == "True" || token == "TRUE") {
                    return node_type::BOOLEAN;
                }
                break;
            case '.':
                if (token == ".inf" || token == ".Inf" || token == ".INF" || token == ".nan" || token == ".NaN" ||
                    token == ".NAN") {
                    return node_type::FLOAT;
                }
                break;
            }
            break;
        case 5:
            switch (token[0]) {
            case 'f':
            case 'F':
                if (token == "false" || token == "False" || token == "FALSE") {
                    return node_type::BOOLEAN;
                }
                break;
            case '-':
                if (token[1] == '.' && (token == "-.inf" || token == "-.Inf" || token == "-.INF")) {
                    return node_type::FLOAT;
                }
                break;
            }
            break;
        }

        return scan_possible_number_token(token);
    }

private:
    static node_type scan_possible_number_token(const std::string& token) {
        std::string::const_iterator itr = token.begin();
        std::size_t size = token.size();
        FK_YAML_ASSERT(size > 0);

        switch (*itr) {
        case '-':
            return (size > 1) ? scan_negative_number(++itr, --size) : node_type::STRING;
        case '+':
            return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::STRING;
        case '0':
            return (size > 1) ? scan_after_zero_at_first(++itr, --size) : node_type::INTEGER;
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
            return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::INTEGER;
        default:
            return node_type::STRING;
        }
    }

    static node_type scan_negative_number(std::string::const_iterator itr, std::size_t size) {
        FK_YAML_ASSERT(size > 0);

        if (is_digit(*itr)) {
            return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::INTEGER;
        }

        return node_type::STRING;
    }

    static node_type scan_after_zero_at_first(std::string::const_iterator itr, std::size_t size) {
        FK_YAML_ASSERT(size > 0);

        if (is_digit(*itr)) {
            // a token consisting of the beginning '0' and some following numbers, e.g., `0123`, is not an integer
            // according to https://yaml.org/spec/1.2.2/#10213-integer.
            return node_type::STRING;
        }

        switch (*itr) {
        case '.': {
            if (size == 1) {
                // 0 is omitted after `0.`.
                return node_type::FLOAT;
            }
            node_type ret = scan_after_decimal_point(++itr, --size, true);
            return (ret == node_type::STRING) ? node_type::STRING : node_type::FLOAT;
        }
        case 'o':
            return (size > 1) ? scan_octal_number(++itr, --size) : node_type::STRING;
        case 'x':
            return (size > 1) ? scan_hexadecimal_number(++itr, --size) : node_type::STRING;
        default:
            return node_type::STRING;
        }
    }

    static node_type scan_decimal_number(std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {
        FK_YAML_ASSERT(size > 0);

        if (is_digit(*itr)) {
            return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::INTEGER;
        }

        switch (*itr) {
        case '.': {
            if (has_decimal_point) {
                // the token has more than one period, e.g., a semantic version `1.2.3`.
                return node_type::STRING;
            }
            if (size == 1) {
                // 0 is omitted after the decimal point
                return node_type::FLOAT;
            }
            node_type ret = scan_after_decimal_point(++itr, --size, true);
            return (ret == node_type::STRING) ? node_type::STRING : node_type::FLOAT;
        }
        case 'e':
        case 'E':
            return (size > 1) ? scan_after_exponent(++itr, --size, has_decimal_point) : node_type::STRING;
        default:
            return node_type::STRING;
        }
    }

    static node_type scan_after_decimal_point(
        std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {
        FK_YAML_ASSERT(size > 0);

        if (is_digit(*itr)) {
            return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::FLOAT;
        }

        return node_type::STRING;
    }

    static node_type scan_after_exponent(std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {
        FK_YAML_ASSERT(size > 0);

        if (is_digit(*itr)) {
            return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::FLOAT;
        }

        switch (*itr) {
        case '+':
        case '-':
            return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::STRING;
        default:
            return node_type::STRING;
        }
    }

    static node_type scan_octal_number(std::string::const_iterator itr, std::size_t size) {
        FK_YAML_ASSERT(size > 0);

        switch (*itr) {
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
            return (size > 1) ? scan_octal_number(++itr, --size) : node_type::INTEGER;
        default:
            return node_type::STRING;
        }
    }

    static node_type scan_hexadecimal_number(std::string::const_iterator itr, std::size_t size) {
        FK_YAML_ASSERT(size > 0);

        if (is_xdigit(*itr)) {
            return (size > 1) ? scan_hexadecimal_number(++itr, --size) : node_type::INTEGER;
        }
        return node_type::STRING;
    }
};

FK_YAML_DETAIL_NAMESPACE_END

#endif /* FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_ */

1	/// _______ __ __ __ _____ __ __ __
2	/// \| __\| \|_/ \| \_/ \|/ _ \ / \/ \\| \| fkYAML: A C++ header-only YAML library
3	/// \| __\| _ < \_ _/\| ___ \| _ \| \|___ version 0.3.11
4	/// \|__\| \|_\| \__\| \|_\| \|_\| \|_\|___\|\|___\|______\| https://github.com/fktn-k/fkYAML
5	///
6	/// SPDX-FileCopyrightText: 2023-2024 Kensuke Fukutani <fktn.dev@gmail.com>
7	/// SPDX-License-Identifier: MIT
8	///
9	/// @file
10
11	#ifndef FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_
12	#define FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_
13
14	#include <algorithm>
15	#include <string>
16
17	#include <fkYAML/detail/macros/version_macros.hpp>
18	#include <fkYAML/detail/assert.hpp>
19	#include <fkYAML/node_type.hpp>
20
21	FK_YAML_DETAIL_NAMESPACE_BEGIN
22
23	namespace {
24
25	/// @brief Check if the given character is a digit.
26	/// @note This function is needed to avoid assertion failures in `std::isdigit()` especially when compiled with MSVC.
27	/// @param c A character to be checked.
28	/// @return true if the given character is a digit, false otherwise.
29	inline bool is_digit(char c) {	412✔
30	return ('0' <= c && c <= '9');	412✔
31	}
32
33	/// @brief Check if the given character is a hex-digit.
34	/// @note This function is needed to avoid assertion failures in `std::isxdigit()` especially when compiled with MSVC.
35	/// @param c A character to be checked.
36	/// @return true if the given character is a hex-digit, false otherwise.
37	inline bool is_xdigit(char c) {	89✔
38	return (('0' <= c && c <= '9') \|\| ('A' <= c && c <= 'F') \|\| ('a' <= c && c <= 'f'));	89✔
39	}
40
41	} // namespace
42	class scalar_scanner {
43	public:
44	static node_type scan(const std::string& token) {	892✔
45	switch (token.size()) {	892✔
46	case 0:	1✔
47	return node_type::STRING;
48	case 1:
49	if (token[0] == '~') {	891✔
50	return node_type::NULL_OBJECT;	891✔
51	}	108✔
52	break;
53	case 4:
54	switch (token[0]) {	783✔
55	case 'n':
56	case 'N':	783✔
57	if (token == "null" \|\| token == "Null" \|\| token == "NULL") {	33✔
58	return node_type::NULL_OBJECT;	33✔
59	}	2✔
60	break;
61	case 't':	31✔
62	case 'T':	254✔
63	if (token == "true" \|\| token == "True" \|\| token == "TRUE") {	254✔
64	return node_type::BOOLEAN;	27✔
65	}
66	break;	27✔
67	case '.':	11✔
68	if (token == ".inf" \|\| token == ".Inf" \|\| token == ".INF" \|\| token == ".nan" \|\| token == ".NaN" \|\|
69	token == ".NAN") {	2✔
70	return node_type::FLOAT;	13✔
71	}	11✔
72	break;	110✔
73	}
74	break;	110✔
75	case 5:	6✔
76	switch (token[0]) {
77	case 'f':	3✔
78	case 'F':	9✔
79	if (token == "false" \|\| token == "False" \|\| token == "FALSE") {	6✔
80	return node_type::BOOLEAN;	16✔
81	}	16✔
82	break;	16✔
83	case '-':	12✔
84	if (token[1] == '.' && (token == "-.inf" \|\| token == "-.Inf" \|\| token == "-.INF")) {	10✔
85	return node_type::FLOAT;	28✔
86	}	16✔
87	break;	15✔
88	}
89	break;	1✔
90	}
91
92	return scan_possible_number_token(token);	85✔
93	}	66✔
94		66✔
95	private:	13✔
96	static node_type scan_possible_number_token(const std::string& token) {	13✔
97	std::string::const_iterator itr = token.begin();	12✔
98	std::size_t size = token.size();
99	FK_YAML_ASSERT(size > 0);	1✔
100		5✔
101	switch (*itr) {	5✔
102	case '-':	5✔
103	return (size > 1) ? scan_negative_number(++itr, --size) : node_type::STRING;	5!
104	case '+':	5✔
105	return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::STRING;
UNCOV 106	case '0':	×
107	return (size > 1) ? scan_after_zero_at_first(++itr, --size) : node_type::INTEGER;
108	case '1':	8✔
109	case '2':	8✔
110	case '3':	7✔
111	case '4':	19✔
112	case '5':	10✔
113	case '6':	3✔
114	case '7':	7✔
115	case '8':	6✔
116	case '9':
117	return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::INTEGER;
118	default:	2✔
119	return node_type::STRING;
120	}	43✔
121	}
122
123	static node_type scan_negative_number(std::string::const_iterator itr, std::size_t size) {	589✔
124	FK_YAML_ASSERT(size > 0);
125
126	if (is_digit(*itr)) {
127	return (size > 1) ? scan_decimal_number(++itr, --size, false) : node_type::INTEGER;	697✔
128	}	697✔
129
130	return node_type::STRING;	697✔
131	}	13✔
132		13✔
133	static node_type scan_after_zero_at_first(std::string::const_iterator itr, std::size_t size) {	1✔
134	FK_YAML_ASSERT(size > 0);	1✔
135		43✔
136	if (is_digit(*itr)) {	43✔
137	// a token consisting of the beginning '0' and some following numbers, e.g., `0123`, is not an integer	129✔
138	// according to https://yaml.org/spec/1.2.2/#10213-integer.
139	return node_type::STRING;
140	}
141
142	switch (*itr) {
143	case '.': {
144	if (size == 1) {
145	// 0 is omitted after `0.`.
146	return node_type::FLOAT;	129✔
147	}	511✔
148	node_type ret = scan_after_decimal_point(++itr, --size, true);	511✔
149	return (ret == node_type::STRING) ? node_type::STRING : node_type::FLOAT;
150	}
151	case 'o':
152	return (size > 1) ? scan_octal_number(++itr, --size) : node_type::STRING;	13✔
153	case 'x':	13✔
154	return (size > 1) ? scan_hexadecimal_number(++itr, --size) : node_type::STRING;
155	default:	13✔
156	return node_type::STRING;	9✔
157	}
158	}
159		4✔
160	static node_type scan_decimal_number(std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {
161	FK_YAML_ASSERT(size > 0);
162		42✔
163	if (is_digit(*itr)) {	42✔
164	return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::INTEGER;
165	}	42✔
166
167	switch (*itr) {
168	case '.': {	1✔
169	if (has_decimal_point) {
170	// the token has more than one period, e.g., a semantic version `1.2.3`.
171	return node_type::STRING;	41✔
172	}	5✔
173	if (size == 1) {	5✔
174	// 0 is omitted after the decimal point
175	return node_type::FLOAT;	1✔
176	}
177	node_type ret = scan_after_decimal_point(++itr, --size, true);	4✔
178	return (ret == node_type::STRING) ? node_type::STRING : node_type::FLOAT;	4✔
179	}
180	case 'e':	6✔
181	case 'E':	6✔
182	return (size > 1) ? scan_after_exponent(++itr, --size, has_decimal_point) : node_type::STRING;	29✔
183	default:	29✔
184	return node_type::STRING;	1✔
185	}	1✔
186	}
187
188	static node_type scan_after_decimal_point(
189	std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {	300✔
190	FK_YAML_ASSERT(size > 0);	300✔
191
192	if (is_digit(*itr)) {	300✔
193	return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::FLOAT;	241✔
194	}
195
196	return node_type::STRING;	59✔
197	}	50✔
198		50✔
199	static node_type scan_after_exponent(std::string::const_iterator itr, std::size_t size, bool has_decimal_point) {
200	FK_YAML_ASSERT(size > 0);	2✔
201
202	if (is_digit(*itr)) {	48✔
203	return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::FLOAT;
204	}	2✔
205
206	switch (*itr) {	46✔
207	case '+':	46✔
208	case '-':
209	return (size > 1) ? scan_decimal_number(++itr, --size, has_decimal_point) : node_type::STRING;	8✔
210	default:
211	return node_type::STRING;	8✔
212	}	1✔
213	}	1✔
214
215	static node_type scan_octal_number(std::string::const_iterator itr, std::size_t size) {
216	FK_YAML_ASSERT(size > 0);
217		50✔
218	switch (*itr) {	50✔
219	case '0':
220	case '1':	50✔
221	case '2':	49✔
222	case '3':
223	case '4':
224	case '5':	1✔
225	case '6':
226	case '7':
227	return (size > 1) ? scan_octal_number(++itr, --size) : node_type::INTEGER;	7✔
228	default:	7✔
229	return node_type::STRING;
230	}	7✔
231	}	1✔
232
233	static node_type scan_hexadecimal_number(std::string::const_iterator itr, std::size_t size) {
234	FK_YAML_ASSERT(size > 0);	6✔
235		5✔
236	if (is_xdigit(*itr)) {
237	return (size > 1) ? scan_hexadecimal_number(++itr, --size) : node_type::INTEGER;	5✔
238	}	1✔
239	return node_type::STRING;	1✔
240	}
241	};
242
243	FK_YAML_DETAIL_NAMESPACE_END	14✔
244		14✔
245	#endif /* FK_YAML_DETAIL_INPUT_SCALAR_SCANNER_HPP_ */

fktn-k / fkYAML / 10749342211

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous