17095454925

Committed 20 Aug 2025 10:15AM UTC coverage: 87.175% (-0.3%) from 87.426%

Build # 17095454925

Build Type

Pull #571

github

Committed by

web-flow

Commit Message

Merge 582ae8a8d into 120045f7e

Pull Request Pull Request #571: Feat/better parser position tracking

Run Details

594 of 697 new or added lines in 17 files covered. (85.22%)

5 existing lines in 2 files now uncovered.

7545 of 8655 relevant lines covered (87.18%)

129898.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.63

/src/arkreactor/Compiler/AST/BaseParser.cpp

#include <Ark/Compiler/AST/BaseParser.hpp>
#include <Ark/Error/Exceptions.hpp>

#include <utility>
#include <algorithm>

#include <fmt/core.h>

namespace Ark::internal
{
    void BaseParser::registerNewLine(std::string::iterator it, std::size_t row)
    {
        // search for an existing new line position
        if (std::ranges::find_if(m_it_to_row, [it](const auto& pair) {
                return pair.first == it;
            }) != m_it_to_row.end())
            return;

        // if the mapping is empty, the loop while never hit, and we'll never insert anything
        if (m_it_to_row.empty())
        {
            m_it_to_row.emplace_back(it, row);
            return;
        }

        for (std::size_t i = 0, end = m_it_to_row.size(); i < end; ++i)
        {
            auto current_it = m_it_to_row[i].first;
            auto next_it = i + 1 < end ? m_it_to_row[i + 1].first : m_str.end();
            if (current_it < it && it < next_it)
            {
                m_it_to_row.insert(
                    m_it_to_row.begin() + static_cast<decltype(m_it_to_row)::difference_type>(i) + 1,
                    std::make_pair(it, row));
                break;
            }
        }
    }

    void BaseParser::next()
    {
        m_it = m_next_it;
        if (isEOF())
        {
            m_sym = utf8_char_t();  // reset sym to EOF
            return;
        }

        // getting a character from the stream
        auto [it, sym] = utf8_char_t::at(m_it, m_str.end());
        m_next_it = it;
        m_sym = sym;

        m_previous_filepos = m_filepos;

        if (*m_it == '\n')
        {
            ++m_filepos.row;
            m_filepos.col = 0;
            registerNewLine(m_it, m_filepos.row);
        }
        else if (m_sym.isPrintable())
            m_filepos.col += m_sym.size();
    }

    void BaseParser::initParser(const std::string& filename, const std::string& code)
    {
        m_filename = filename;

        // if the input string is empty, raise an error
        if (code.empty())
        {
            m_sym = utf8_char_t();
            error("Expected symbol, got empty string", m_filepos);
        }

        m_str = code;
        m_it = m_next_it = m_str.begin();

        // otherwise, get the first symbol
        next();
    }

    void BaseParser::backtrack(const long n)
    {
        if (std::cmp_less(n, m_str.size()))
            m_it = m_str.begin() + n;
        else
            return;

        auto [it, sym] = utf8_char_t::at(m_it, m_str.end());
        m_next_it = it;
        m_sym = sym;

        // search for the nearest it < m_it in the map to know the line number
        for (const auto& [at, line] : m_it_to_row)
        {
            if (it <= at)
            {
                m_filepos.row = line - 1;
                break;
            }
        }
        // compute the position in the line
        const auto it_pos = static_cast<std::size_t>(std::distance(m_str.begin(), m_it));
        const std::string_view view { m_str.begin(), m_it };
        const auto nearest_newline_index = view.find_last_of('\n');
        if (nearest_newline_index != std::string_view::npos)
            m_filepos.col = it_pos - nearest_newline_index;
        else
            m_filepos.col = it_pos + 1;
        // We can say that the previous position is the current one, as there isn't anything usable right now
        // Which means we will have to use accept()/next(), which will move filepos and previous_filepos correctly
        m_previous_filepos = m_filepos;
    }

    FilePosition BaseParser::getCursor() const
    {
        return m_filepos;
    }

    FilePosition BaseParser::getPreviousCursor() const
    {
        return m_previous_filepos;
    }

    CodeErrorContext BaseParser::generateErrorContextAtCurrentPosition() const
    {
        const auto [row, col] = getCursor();

        return CodeErrorContext(
            m_filename,
            // for additional contexts, the end position is useless
            FileSpan { .start = FilePos { .line = row, .column = col }, .end = std::nullopt });
    }

    void BaseParser::error(const std::string& error, const FilePosition start_at, const std::optional<CodeErrorContext>& additional_context) const
    {
        const auto [row, col] = getCursor();
        throw CodeError(
            error,
            CodeErrorContext(
                m_filename,
                FileSpan {
                    .start = FilePos { .line = start_at.row, .column = start_at.col },
                    .end = FilePos { .line = row, .column = col } }),
            additional_context);
    }

    void BaseParser::errorWithNextToken(const std::string& message, const std::optional<CodeErrorContext>& additional_context)
    {
        const auto filepos = getCursor();

        anyUntil(IsEither(IsInlineSpace, IsEither(IsChar('('), IsChar(')'))));
        error(message, filepos, additional_context);
    }

    void BaseParser::expectSuffixOrError(const char suffix, const std::string& context, const std::optional<CodeErrorContext>& additional_context)
    {
        if (!accept(IsChar(suffix)))
            errorWithNextToken(fmt::format("Missing '{}' {}", suffix, context), additional_context);
    }

    bool BaseParser::accept(const CharPred& t, std::string* s)
    {
        if (isEOF())
            return false;

        // return false if the predicate couldn't consume the symbol
        if (!t(m_sym.codepoint()))
            return false;
        // otherwise, add it to the string and go to the next symbol
        if (s != nullptr)
            *s += m_sym.c_str();

        next();
        return true;
    }

    bool BaseParser::expect(const CharPred& t, std::string* s)
    {
        // throw an error if the predicate couldn't consume the symbol
        if (!t(m_sym.codepoint()))
            error("Expected " + t.name, getCursor());
        // otherwise, add it to the string and go to the next symbol
        if (s != nullptr)
            *s += m_sym.c_str();
        next();
        return true;
    }

    std::string BaseParser::peek() const
    {
        return m_sym.c_str();
    }

    bool BaseParser::space(std::string* s)
    {
        if (accept(IsSpace))
        {
            if (s != nullptr)
                s->push_back(' ');
            // loop while there are still ' ' to consume
            while (accept(IsSpace))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::inlineSpace(std::string* s)
    {
        if (accept(IsInlineSpace))
        {
            if (s != nullptr)
                s->push_back(' ');
            // loop while there are still ' ' to consume
            while (accept(IsInlineSpace))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::comment(std::string* s)
    {
        if (accept(IsChar('#'), s))
        {
            while (accept(IsNot(IsChar('\n')), s))
                ;
            accept(IsChar('\n'), s);
            return true;
        }
        return false;
    }

    std::string BaseParser::spaceComment()
    {
        std::string s;

        inlineSpace();
        while (!isEOF() && comment(&s))
            inlineSpace();

        return s;
    }

    std::string BaseParser::newlineOrComment()
    {
        std::string s;

        space();
        while (!isEOF() && comment(&s))
            space();

        return s;
    }

    bool BaseParser::prefix(const char c)
    {
        if (!accept(IsChar(c)))
            return false;
        return true;
    }

    bool BaseParser::number(std::string* s)
    {
        if (accept(IsDigit, s))
        {
            // consume all the digits available,
            // stop when the symbol isn't a digit anymore
            while (accept(IsDigit, s))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::signedNumber(std::string* s)
    {
        accept(IsMinus, s);
        if (!number(s))
            return false;

        // (optional) floating part
        accept(IsChar('.'), s) && number(s);
        // (optional) scientific part
        if (accept(IsEither(IsChar('e'), IsChar('E')), s))
        {
            accept(IsEither(IsMinus, IsChar('+')), s);
            number(s);
        }

        return true;
    }

    bool BaseParser::hexNumber(unsigned int length, std::string* s)
    {
        while (length != 0)
        {
            if (!accept(IsHex, s))
                return false;
            --length;
        }
        return true;
    }

    bool BaseParser::name(std::string* s)
    {
        const auto alpha_symbols = IsEither(IsAlpha, IsSymbol);
        const auto alnum_symbols = IsEither(IsAlnum, IsSymbol);

        if (accept(alpha_symbols, s))
        {
            while (accept(alnum_symbols, s))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::sequence(const std::string& s)
    {
        return std::ranges::all_of(s, [this](const char c) {
            return accept(IsChar(c));
        });
    }

    bool BaseParser::packageName(std::string* s)
    {
        if (accept(IsAlnum, s))
        {
            while (accept(IsEither(IsAlnum, IsEither(IsChar('_'), IsChar('-'))), s))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::anyUntil(const CharPred& delim, std::string* s)
    {
        if (accept(IsNot(delim), s))
        {
            while (accept(IsNot(delim), s))
                ;
            return true;
        }
        return false;
    }

    bool BaseParser::oneOf(const std::initializer_list<std::string> words, std::string* s)
    {
        std::string buffer;
        if (!name(&buffer))
            return false;

        if (s)
            *s = buffer;

        return std::ranges::any_of(words, [&buffer](const std::string& word) {
            return word == buffer;
        });
    }
}

1	#include <Ark/Compiler/AST/BaseParser.hpp>
2	#include <Ark/Error/Exceptions.hpp>
3
4	#include <utility>
5	#include <algorithm>
6
7	#include <fmt/core.h>
8
9	namespace Ark::internal
10	{
11	void BaseParser::registerNewLine(std::string::iterator it, std::size_t row)	31,488✔
12	{	31,488✔
13	// search for an existing new line position
14	if (std::ranges::find_if(m_it_to_row, [it](const auto& pair) {	6,189,909✔
15	return pair.first == it;	6,158,421✔
16	}) != m_it_to_row.end())	31,488✔
17	return;	1,414✔
18
19	// if the mapping is empty, the loop while never hit, and we'll never insert anything
20	if (m_it_to_row.empty())	30,074✔
21	{
22	m_it_to_row.emplace_back(it, row);	468✔
23	return;	468✔
24	}
25
26	for (std::size_t i = 0, end = m_it_to_row.size(); i < end; ++i)	5,989,861✔
27	{
28	auto current_it = m_it_to_row[i].first;	5,960,255✔
29	auto next_it = i + 1 < end ? m_it_to_row[i + 1].first : m_str.end();	5,960,255✔
30	if (current_it < it && it < next_it)	5,960,255✔
31	{
32	m_it_to_row.insert(	59,212✔
33	m_it_to_row.begin() + static_cast<decltype(m_it_to_row)::difference_type>(i) + 1,	29,606✔
34	std::make_pair(it, row));	29,606✔
35	break;	29,606✔
36	}
37	}	5,960,255✔
38	}	31,488✔
39
40	void BaseParser::next()	1,773,067✔
41	{	1,773,067✔
42	m_it = m_next_it;	1,773,067✔
43	if (isEOF())	1,773,067✔
44	{
45	m_sym = utf8_char_t(); // reset sym to EOF	492✔
46	return;	492✔
47	}
48
49	// getting a character from the stream
50	auto [it, sym] = utf8_char_t::at(m_it, m_str.end());	1,772,575✔
51	m_next_it = it;	1,772,575✔
52	m_sym = sym;	1,772,575✔
53
54	m_previous_filepos = m_filepos;	1,772,575✔
55
56	if (*m_it == '\n')	1,772,575✔
57	{
58	++m_filepos.row;	31,488✔
59	m_filepos.col = 0;	31,488✔
60	registerNewLine(m_it, m_filepos.row);	31,488✔
61	}	31,488✔
62	else if (m_sym.isPrintable())	1,741,087✔
63	m_filepos.col += m_sym.size();	1,741,087✔
64	}	1,773,067✔
65
66	void BaseParser::initParser(const std::string& filename, const std::string& code)	512✔
67	{	512✔
68	m_filename = filename;	512✔
69
70	// if the input string is empty, raise an error
71	if (code.empty())	512✔
72	{
73	m_sym = utf8_char_t();	1✔
74	error("Expected symbol, got empty string", m_filepos);	1✔
75	}	×
76
77	m_str = code;	511✔
78	m_it = m_next_it = m_str.begin();	511✔
79
80	// otherwise, get the first symbol
81	next();	511✔
82	}	512✔
83
84	void BaseParser::backtrack(const long n)	621,853✔
85	{	621,853✔
86	if (std::cmp_less(n, m_str.size()))	621,853✔
87	m_it = m_str.begin() + n;	621,853✔
88	else
NEW 89	return;	×
90
91	auto [it, sym] = utf8_char_t::at(m_it, m_str.end());	128,937,503✔
92	m_next_it = it;	621,853✔
93	m_sym = sym;	621,853✔
94
95	// search for the nearest it < m_it in the map to know the line number
96	for (const auto& [at, line] : m_it_to_row)	128,939,183✔
97	{
98	if (it <= at)	128,315,650✔
99	{
100	m_filepos.row = line - 1;	1,680✔
101	break;	1,680✔
102	}
103	}	128,315,650✔
104	// compute the position in the line
105	const auto it_pos = static_cast<std::size_t>(std::distance(m_str.begin(), m_it));	621,853✔
106	const std::string_view view { m_str.begin(), m_it };	621,853✔
107	const auto nearest_newline_index = view.find_last_of('\n');	621,853✔
108	if (nearest_newline_index != std::string_view::npos)	621,853✔
109	m_filepos.col = it_pos - nearest_newline_index;	610,571✔
110	else
111	m_filepos.col = it_pos + 1;	11,282✔
112	// We can say that the previous position is the current one, as there isn't anything usable right now
113	// Which means we will have to use accept()/next(), which will move filepos and previous_filepos correctly
114	m_previous_filepos = m_filepos;	621,853✔
115	}	621,853✔
116
117	FilePosition BaseParser::getCursor() const	881,230✔
118	{	881,230✔
119	return m_filepos;	881,230✔
120	}
121
NEW 122	FilePosition BaseParser::getPreviousCursor() const	×
NEW 123	{	×
NEW 124	return m_previous_filepos;	×
125	}
126
127	CodeErrorContext BaseParser::generateErrorContextAtCurrentPosition() const	327,734✔
128	{	327,734✔
129	const auto [row, col] = getCursor();	327,734✔
130
131	return CodeErrorContext(	327,734✔
132	m_filename,	327,734✔
133	// for additional contexts, the end position is useless
134	FileSpan { .start = FilePos { .line = row, .column = col }, .end = std::nullopt });	983,202✔
135	}	327,734✔
136
137	void BaseParser::error(const std::string& error, const FilePosition start_at, const std::optional<CodeErrorContext>& additional_context) const	45✔
138	{	45✔
139	const auto [row, col] = getCursor();	135✔
140	throw CodeError(	135✔
141	error,	45✔
142	CodeErrorContext(	45✔
143	m_filename,	45✔
144	FileSpan {	45✔
145	.start = FilePos { .line = start_at.row, .column = start_at.col },	45✔
146	.end = FilePos { .line = row, .column = col } }),	135✔
147	additional_context);	45✔
148	}	90✔
149
150	void BaseParser::errorWithNextToken(const std::string& message, const std::optional<CodeErrorContext>& additional_context)	37✔
151	{	37✔
152	const auto filepos = getCursor();	37✔
153
154	anyUntil(IsEither(IsInlineSpace, IsEither(IsChar('('), IsChar(')'))));	37✔
155	error(message, filepos, additional_context);	37✔
156	}	37✔
157
158	void BaseParser::expectSuffixOrError(const char suffix, const std::string& context, const std::optional<CodeErrorContext>& additional_context)	24,707✔
159	{	24,707✔
160	if (!accept(IsChar(suffix)))	24,716✔
161	errorWithNextToken(fmt::format("Missing '{}' {}", suffix, context), additional_context);	9✔
162	}	24,707✔
163
164	bool BaseParser::accept(const CharPred& t, std::string* s)	3,562,345✔
165	{	3,562,345✔
166	if (isEOF())	3,562,345✔
167	return false;	560✔
168
169	// return false if the predicate couldn't consume the symbol
170	if (!t(m_sym.codepoint()))	3,561,785✔
171	return false;	1,791,624✔
172	// otherwise, add it to the string and go to the next symbol
173	if (s != nullptr)	1,770,161✔
174	*s += m_sym.c_str();	1,405,271✔
175
176	next();	1,770,161✔
177	return true;	1,770,161✔
178	}	3,562,345✔
179
180	bool BaseParser::expect(const CharPred& t, std::string* s)	2,395✔
181	{	2,395✔
182	// throw an error if the predicate couldn't consume the symbol
183	if (!t(m_sym.codepoint()))	2,395✔
NEW 184	error("Expected " + t.name, getCursor());	×
185	// otherwise, add it to the string and go to the next symbol
186	if (s != nullptr)	2,395✔
187	*s += m_sym.c_str();	×
188	next();	2,395✔
189	return true;	2,395✔
190	}	×
191
192	std::string BaseParser::peek() const	5✔
193	{	5✔
194	return m_sym.c_str();	5✔
195	}	×
196
197	bool BaseParser::space(std::string* s)	331,280✔
198	{	331,280✔
199	if (accept(IsSpace))	331,280✔
200	{
201	if (s != nullptr)	120,902✔
202	s->push_back(' ');	×
203	// loop while there are still ' ' to consume
204	while (accept(IsSpace))	162,550✔
205	;
206	return true;	120,902✔
207	}
208	return false;	210,378✔
209	}	331,280✔
210
211	bool BaseParser::inlineSpace(std::string* s)	13,411✔
212	{	13,411✔
213	if (accept(IsInlineSpace))	13,411✔
214	{
215	if (s != nullptr)	890✔
216	s->push_back(' ');	×
217	// loop while there are still ' ' to consume
218	while (accept(IsInlineSpace))	899✔
219	;
220	return true;	890✔
221	}
222	return false;	12,521✔
223	}	13,411✔
224
225	bool BaseParser::comment(std::string* s)	343,864✔
226	{	343,864✔
227	if (accept(IsChar('#'), s))	343,864✔
228	{
229	while (accept(IsNot(IsChar('\n')), s))	470,295✔
230	;
231	accept(IsChar('\n'), s);	14,535✔
232	return true;	14,535✔
233	}
234	return false;	329,329✔
235	}	343,864✔
236
237	std::string BaseParser::spaceComment()	13,372✔
238	{	13,372✔
239	std::string s;	13,372✔
240
241	inlineSpace();	13,372✔
242	while (!isEOF() && comment(&s))	13,411✔
243	inlineSpace();	39✔
244
245	return s;	13,372✔
246	}	13,372✔
247
248	std::string BaseParser::newlineOrComment()	316,465✔
249	{	316,465✔
250	std::string s;	316,465✔
251
252	space();	316,465✔
253	while (!isEOF() && comment(&s))	330,961✔
254	space();	14,496✔
255
256	return s;	316,465✔
257	}	316,465✔
258
259	bool BaseParser::prefix(const char c)	194,789✔
260	{	194,789✔
261	if (!accept(IsChar(c)))	194,789✔
262	return false;	100,964✔
263	return true;	93,825✔
264	}	194,789✔
265
266	bool BaseParser::number(std::string* s)	133,446✔
267	{	133,446✔
268	if (accept(IsDigit, s))	133,446✔
269	{
270	// consume all the digits available,
271	// stop when the symbol isn't a digit anymore
272	while (accept(IsDigit, s))	71,240✔
273	;
274	return true;	70,593✔
275	}
276	return false;	62,853✔
277	}	133,446✔
278
279	bool BaseParser::signedNumber(std::string* s)	133,358✔
280	{	133,358✔
281	accept(IsMinus, s);	133,358✔
282	if (!number(s))	133,358✔
283	return false;	62,853✔
284
285	// (optional) floating part
286	accept(IsChar('.'), s) && number(s);	70,505✔
287	// (optional) scientific part
288	if (accept(IsEither(IsChar('e'), IsChar('E')), s))	70,505✔
289	{
290	accept(IsEither(IsMinus, IsChar('+')), s);	5✔
291	number(s);	5✔
292	}	5✔
293
294	return true;	70,505✔
295	}	133,358✔
296
297	bool BaseParser::hexNumber(unsigned int length, std::string* s)	8✔
298	{	8✔
299	while (length != 0)	56✔
300	{
301	if (!accept(IsHex, s))	48✔
302	return false;	×
303	--length;	48✔
304	}
305	return true;	8✔
306	}	8✔
307
308	bool BaseParser::name(std::string* s)	267,749✔
309	{	267,749✔
310	const auto alpha_symbols = IsEither(IsAlpha, IsSymbol);	267,749✔
311	const auto alnum_symbols = IsEither(IsAlnum, IsSymbol);	267,749✔
312
313	if (accept(alpha_symbols, s))	267,749✔
314	{
315	while (accept(alnum_symbols, s))	825,998✔
316	;
317	return true;	181,524✔
318	}
319	return false;	86,225✔
320	}	267,749✔
321
322	bool BaseParser::sequence(const std::string& s)	61,360✔
323	{	61,360✔
324	return std::ranges::all_of(s, [this](const char c) {	122,866✔
325	return accept(IsChar(c));	61,506✔
326	});	×
327	}
328
329	bool BaseParser::packageName(std::string* s)	367✔
330	{	367✔
331	if (accept(IsAlnum, s))	367✔
332	{
333	while (accept(IsEither(IsAlnum, IsEither(IsChar('_'), IsChar('-'))), s))	2,010✔
334	;
335	return true;	364✔
336	}
337	return false;	3✔
338	}	367✔
339
340	bool BaseParser::anyUntil(const CharPred& delim, std::string* s)	37✔
341	{	37✔
342	if (accept(IsNot(delim), s))	37✔
343	{
344	while (accept(IsNot(delim), s))	1,267✔
345	;
346	return true;	9✔
347	}
348	return false;	28✔
349	}	37✔
350
351	bool BaseParser::oneOf(const std::initializer_list<std::string> words, std::string* s)	132,423✔
352	{	132,423✔
353	std::string buffer;	132,423✔
354	if (!name(&buffer))	132,423✔
355	return false;	9,439✔
356
357	if (s)	122,984✔
358	*s = buffer;	21,964✔
359
360	return std::ranges::any_of(words, [&buffer](const std::string& word) {	282,156✔
361	return word == buffer;	159,172✔
362	});
363	}	132,423✔
364	}

ArkScript-lang / Ark / 17095454925

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous