30035549451

Committed 23 Jul 2026 06:51PM UTC coverage: 92.055% (+1.0%) from 91.09%

Build # 30035549451

Build Type

push

github

Committed by

web-flow

Commit Message

feat(scanner): C-ish tokens, keywords, and string-literal decode (#61)

Expand scanner.lex with C punctuation (. -> ? : ...), hex/float forms,
richer char escapes, and a fuller keyword set (const, struct, sizeof, …)
so later grammar/AST work can see the right terminals.

Add util::StringLiteralDecode for string/escape decoding (unit-tested)
and ScannerTokens tests that pin the new lexemes without depending on
typedef registries or attribute filtering.

Coverage Stats

63 of 86 new or added lines in 1 file covered. (73.26%)

5782 of 6281 relevant lines covered (92.06%)

199864.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.26

/src/util/StringLiteralDecode.cpp

#include "StringLiteralDecode.h"

#include <sstream>

namespace util {

std::vector<unsigned char> decodeStringLiteralBytes(const std::string &token) {
    std::string body = token;
    if (body.size() >= 2 && body.front() == '"' && body.back() == '"') {
        body = body.substr(1, body.size() - 2);
    }
    std::vector<unsigned char> bytes;
    for (std::size_t i = 0; i < body.size(); ++i) {
        if (body[i] == '\\' && i + 1 < body.size()) {
            ++i;
            char e = body[i];
            switch (e) {
            case 'n':
                bytes.push_back('\n');
                break;
            case 't':
                bytes.push_back('\t');
                break;
            case 'r':
                bytes.push_back('\r');
                break;
            case 'a':
                bytes.push_back('\a');
                break;
            case 'b':
                bytes.push_back('\b');
                break;
            case 'f':
                bytes.push_back('\f');
                break;
            case 'v':
                bytes.push_back('\v');
                break;
            case '\\':
                bytes.push_back('\\');
                break;
            case '"':
                bytes.push_back('"');
                break;
            case '\'':
                bytes.push_back('\'');
                break;
            case '?':
                bytes.push_back('?');
                break;
            case 'x':
            case 'X': {
                unsigned value = 0;
                bool any = false;
                while (i + 1 < body.size()) {
                    char d = body[i + 1];
                    unsigned digit;
                    if (d >= '0' && d <= '9') {
                        digit = static_cast<unsigned>(d - '0');
                    } else if (d >= 'a' && d <= 'f') {
                        digit = static_cast<unsigned>(d - 'a' + 10);
                    } else if (d >= 'A' && d <= 'F') {
                        digit = static_cast<unsigned>(d - 'A' + 10);
                    } else {
                        break;
                    }
                    any = true;
                    ++i;
                    value = (value << 4) | digit;
                }
                bytes.push_back(any ? static_cast<unsigned char>(value & 0xffu) : 0);
                break;
            }
            default:
                if (e >= '0' && e <= '7') {
                    unsigned value = static_cast<unsigned>(e - '0');
                    int count = 1;
                    while (count < 3 && i + 1 < body.size() && body[i + 1] >= '0' && body[i + 1] <= '7') {
                        ++i;
                        ++count;
                        value = (value << 3) | static_cast<unsigned>(body[i] - '0');
                    }
                    bytes.push_back(static_cast<unsigned char>(value & 0xffu));
                } else {
                    bytes.push_back(static_cast<unsigned char>(e));
                }
                break;
            }
        } else {
            bytes.push_back(static_cast<unsigned char>(body[i]));
        }
    }
    bytes.push_back(0);
    return bytes;
}

int stringLiteralArrayLength(const std::string &token) { return static_cast<int>(decodeStringLiteralBytes(token).size()); }

std::string toNasmDbDirective(const std::string &token) {
    const auto bytes = decodeStringLiteralBytes(token);
    std::ostringstream declaration;
    declaration << "db ";
    for (std::size_t i = 0; i < bytes.size(); ++i) {
        if (i > 0) {
            declaration << ", ";
        }
        declaration << static_cast<unsigned>(bytes[i]);
    }
    return declaration.str();
}

} // namespace util

1	#include "StringLiteralDecode.h"
2
3	#include <sstream>
4
5	namespace util {
6
7	std::vector<unsigned char> decodeStringLiteralBytes(const std::string &token) {	9✔
8	std::string body = token;	9✔
9	if (body.size() >= 2 && body.front() == '"' && body.back() == '"') {	9✔
10	body = body.substr(1, body.size() - 2);	8✔
11	}
12	std::vector<unsigned char> bytes;	9✔
13	for (std::size_t i = 0; i < body.size(); ++i) {	40✔
14	if (body[i] == '\\' && i + 1 < body.size()) {	31✔
15	++i;	10✔
16	char e = body[i];	10✔
17	switch (e) {	10✔
18	case 'n':	2✔
19	bytes.push_back('\n');	2✔
20	break;	2✔
21	case 't':	1✔
22	bytes.push_back('\t');	1✔
23	break;	1✔
NEW 24	case 'r':	×
NEW 25	bytes.push_back('\r');	×
NEW 26	break;	×
NEW 27	case 'a':	×
NEW 28	bytes.push_back('\a');	×
NEW 29	break;	×
NEW 30	case 'b':	×
NEW 31	bytes.push_back('\b');	×
NEW 32	break;	×
NEW 33	case 'f':	×
NEW 34	bytes.push_back('\f');	×
NEW 35	break;	×
NEW 36	case 'v':	×
NEW 37	bytes.push_back('\v');	×
NEW 38	break;	×
39	case '\\':	1✔
40	bytes.push_back('\\');	1✔
41	break;	1✔
42	case '"':	1✔
43	bytes.push_back('"');	1✔
44	break;	1✔
NEW 45	case '\'':	×
NEW 46	bytes.push_back('\'');	×
NEW 47	break;	×
NEW 48	case '?':	×
NEW 49	bytes.push_back('?');	×
NEW 50	break;	×
51	case 'x':	3✔
52	case 'X': {
53	unsigned value = 0;	3✔
54	bool any = false;	3✔
55	while (i + 1 < body.size()) {	9✔
56	char d = body[i + 1];	8✔
57	unsigned digit;
58	if (d >= '0' && d <= '9') {	8✔
59	digit = static_cast<unsigned>(d - '0');	4✔
60	} else if (d >= 'a' && d <= 'f') {	4✔
NEW 61	digit = static_cast<unsigned>(d - 'a' + 10);	×
62	} else if (d >= 'A' && d <= 'F') {	4✔
63	digit = static_cast<unsigned>(d - 'A' + 10);	2✔
64	} else {
65	break;
66	}
67	any = true;	6✔
68	++i;	6✔
69	value = (value << 4) \| digit;	6✔
70	}
71	bytes.push_back(any ? static_cast<unsigned char>(value & 0xffu) : 0);	3✔
72	break;	3✔
73	}
74	default:	2✔
75	if (e >= '0' && e <= '7') {	2✔
76	unsigned value = static_cast<unsigned>(e - '0');	2✔
77	int count = 1;	2✔
78	while (count < 3 && i + 1 < body.size() && body[i + 1] >= '0' && body[i + 1] <= '7') {	4✔
79	++i;	2✔
80	++count;	2✔
81	value = (value << 3) \| static_cast<unsigned>(body[i] - '0');	2✔
82	}
83	bytes.push_back(static_cast<unsigned char>(value & 0xffu));	2✔
84	} else {	2✔
NEW 85	bytes.push_back(static_cast<unsigned char>(e));	×
86	}
87	break;	2✔
88	}
89	} else {
90	bytes.push_back(static_cast<unsigned char>(body[i]));	21✔
91	}
92	}
93	bytes.push_back(0);	9✔
94	return bytes;	18✔
95	}	9✔
96
97	int stringLiteralArrayLength(const std::string &token) { return static_cast<int>(decodeStringLiteralBytes(token).size()); }	2✔
98
99	std::string toNasmDbDirective(const std::string &token) {	2✔
100	const auto bytes = decodeStringLiteralBytes(token);	2✔
101	std::ostringstream declaration;	2✔
102	declaration << "db ";	2✔
103	for (std::size_t i = 0; i < bytes.size(); ++i) {	7✔
104	if (i > 0) {	5✔
105	declaration << ", ";	3✔
106	}
107	declaration << static_cast<unsigned>(bytes[i]);	5✔
108	}
109	return declaration.str();	4✔
110	}	2✔
111
112	} // namespace util

rieske / trans / 30035549451

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous