12846535547

Committed 18 Jan 2025 06:41PM UTC coverage: 75.846% (-1.7%) from 77.556%

Build # 12846535547

Build Type

Pull #274

github

Committed by

web-flow

Commit Message

Merge 8b9494010 into b007cb890

Pull Request Pull Request #274: Eliminate warnings

Run Details

279 of 358 new or added lines in 13 files covered. (77.93%)

38 existing lines in 2 files now uncovered.

1837 of 2422 relevant lines covered (75.85%)

240.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.33

/Parser.cpp

#include "Parser.h"

#include <filesystem>
#include <fstream>

#include "CommonRegexes.h"
#include "Log.h"
#include "ParserHelpers.h"
#include "StringUtils.h"



using std::filesystem::path;
using std::filesystem::u8path;



namespace commonItems
{
std::string getNextLexeme(std::istream& theStream);
} // namespace commonItems


void commonItems::absorbBOM(std::istream& theStream)
{
        if (const auto firstChar = static_cast<char>(theStream.peek()); firstChar == '\xEF')
        {
                char bitBucket[3];
                theStream.read(bitBucket, sizeof bitBucket);
        }
}


void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunction& function)
{
        registeredKeywordStrings.insert(std::make_pair(keyword, function));
}


void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunctionStreamOnly& function)
{
        registeredKeywordStringsStreamOnly.insert(std::make_pair(keyword, function));
}


void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function)
{
        generatedRegexes.emplace_back(std::regex(keyword), function);
}


void commonItems::parser::IgnoreUnregisteredItems()
{
        registerRegex(catchallRegex, ignoreItem);
}


void commonItems::parser::IgnoreAndLogUnregisteredItems()
{
        registerRegex(catchallRegex, ignoreAndLogItem);
}


void commonItems::parser::IgnoreAndStoreUnregisteredItems(std::set<std::string>& ignored_tokens)
{
        registerRegex(catchallRegex, [&ignored_tokens](const std::string& key, std::istream& the_stream) {
                ignored_tokens.insert(key);
                ignoreItem(key, the_stream);
        });
}


void commonItems::parser::parseStream(std::istream& theStream)
{
        auto braceDepth = 0;
        auto value = false; // tracker to indicate whether we reached the value part of key=value pair
        std::string tokensSoFar;

        while (true)
        {
                if (auto token = getNextToken(theStream); token)
                {
                        tokensSoFar += *token;
                        if (*token == "=")
                        {
                                if (!value)
                                {
                                        value = true; // swapping to value part.
                                        continue;
                                }

                                // value is positive, meaning we were at value, and now we're hitting an equal. This is bad. We need to
                                // manually fast-forward to brace-lvl 0 and die.
                                char inputChar;
                                while (braceDepth)
                                {
                                        theStream >> inputChar;
                                        if (inputChar == '{')
                                                braceDepth++;
                                        if (inputChar == '}')
                                                braceDepth--;
                                        if (!isspace(inputChar))
                                                tokensSoFar += inputChar;
                                }
                                Log(LogLevel::Warning) << "Broken token syntax at " << tokensSoFar;
                                return;
                        }
                        if (*token == "{")
                                braceDepth++;
                        else if (*token == "}")
                        {
                                braceDepth--;
                                if (braceDepth == 0)
                                        break;
                        }
                        else
                                Log(LogLevel::Warning) << "Unknown token while parsing stream: " << *token;
                }
                else
                        break;
        }
}


void commonItems::parser::parseFile(path filename)
{
        std::ifstream theFile(filename);
        if (!theFile.is_open())
        {
                Log(LogLevel::Error) << "Could not open " << filename << " for parsing.";
                return;
        }

        absorbBOM(theFile);
        parseStream(theFile);
        theFile.close();
}


void commonItems::parser::parseFile(std::string_view filename)
{
#pragma warning(push)
#pragma warning(disable : 4996)
        parseFile(u8path(filename));
#pragma warning(pop)
}


void commonItems::parser::clearRegisteredKeywords() noexcept
{
        std::map<std::string, parsingFunction>().swap(registeredKeywordStrings);
        std::map<std::string, parsingFunctionStreamOnly>().swap(registeredKeywordStringsStreamOnly);
        std::vector<std::pair<std::regex, parsingFunction>>().swap(generatedRegexes);
}

void commonItems::parser::operator()(std::istream& theStream)
{
        this->parseStream(theStream);
}

std::optional<std::string> commonItems::parser::getNextToken(std::istream& theStream)
{
        theStream >> std::noskipws;
        std::string toReturn;

        auto gotToken = false;
        while (!gotToken)
        {
                if (theStream.eof())
                        return {};

                toReturn = getNextLexeme(theStream);

                const auto strippedLexeme = remQuotes(toReturn);
                const auto isLexemeQuoted = strippedLexeme.size() < toReturn.size();

                auto matched = tryToMatchAgainstKeywords(toReturn, strippedLexeme, isLexemeQuoted, theStream);

                if (!matched)
                        matched = tryToMatchAgainstRegexes(toReturn, strippedLexeme, isLexemeQuoted, theStream);

                if (!matched)
                        gotToken = true;
        }

        if (!toReturn.empty())
                return std::move(toReturn);
        return std::nullopt;
}


inline bool commonItems::parser::tryToMatchAgainstKeywords(const std::string& toReturn,
         const std::string& strippedLexeme,
         bool isLexemeQuoted,
         std::istream& theStream)
{
        if (const auto& match = registeredKeywordStringsStreamOnly.find(toReturn); match != registeredKeywordStringsStreamOnly.end())
        {
                match->second(theStream);
                return true;
        }
        if (const auto& match = registeredKeywordStrings.find(toReturn); match != registeredKeywordStrings.end())
        {
                match->second(toReturn, theStream);
                return true;
        }
        if (isLexemeQuoted)
        {
                if (const auto& strippedMatch = registeredKeywordStringsStreamOnly.find(strippedLexeme); strippedMatch != registeredKeywordStringsStreamOnly.end())
                {
                        strippedMatch->second(theStream);
                        return true;
                }
                if (const auto& strippedMatch = registeredKeywordStrings.find(strippedLexeme); strippedMatch != registeredKeywordStrings.end())
                {
                        strippedMatch->second(toReturn, theStream);
                        return true;
                }
        }

        return false;
}

inline bool commonItems::parser::tryToMatchAgainstRegexes(const std::string& toReturn,
         const std::string& strippedLexeme,
         bool isLexemeQuoted,
         std::istream& theStream)
{
        for (const auto& [regex, parsingFunction]: generatedRegexes)
        {
                if (std::smatch match; std::regex_match(toReturn, match, regex))
                {
                        parsingFunction(toReturn, theStream);
                        return true;
                }
        }
        if (isLexemeQuoted)
        {
                for (const auto& [regex, parsingFunction]: generatedRegexes)
                {
                        if (std::smatch match; std::regex_match(strippedLexeme, match, regex))
                        {
                                parsingFunction(toReturn, theStream);
                                return true;
                        }
                }
        }
        return false;
}


std::optional<std::string> commonItems::parser::getNextTokenWithoutMatching(std::istream& theStream)
{
        theStream >> std::noskipws;

        std::string toReturn;

        auto gotToken = false;
        while (!gotToken)
        {
                if (theStream.eof())
                        return std::nullopt;

                toReturn = getNextLexeme(theStream);
                gotToken = true;
        }

        if (!toReturn.empty())
                return std::move(toReturn);
        return std::nullopt;
}


std::string commonItems::getNextLexeme(std::istream& theStream)
{
        std::string toReturn;

        auto inQuotes = false;
        auto inLiteralQuote = false;
        unsigned char previousCharacter = '\0';

        while (true)
        {
                char inputChar;
                theStream >> inputChar;
                if (theStream.eof())
                        break;
                if (!inQuotes && inputChar == '#')
                {
                        std::string bitBucket;
                        std::getline(theStream, bitBucket);
                        if (!toReturn.empty())
                                break;
                }
                else if (inputChar == '\n')
                {
                        if (!inQuotes)
                        {
                                if (!toReturn.empty())
                                        break;
                        }
                        else
                        {
                                // fix paradox' mistake and don't break proper names in half
                                toReturn += " ";
                        }
                }
                else if (inputChar == '\"' && !inQuotes && toReturn.empty())
                {
                        inQuotes = true;
                        toReturn += inputChar;
                }
                else if (inputChar == '\"' && !inQuotes && toReturn.size() == 1 && toReturn.back() == 'R')
                {
                        inLiteralQuote = true;
                        toReturn.pop_back();
                        toReturn += inputChar;
                }
                else if (inputChar == '(' && inLiteralQuote && toReturn.size() == 1)
                {
                        continue;
                }
                else if (inputChar == '\"' && inLiteralQuote && previousCharacter == ')')
                {
                        toReturn.pop_back();
                        toReturn += inputChar;
                        break;
                }
                else if (inputChar == '\"' && inQuotes && previousCharacter != '\\')
                {
                        toReturn += inputChar;
                        break;
                }
                else if (!inQuotes && !inLiteralQuote && std::isspace(inputChar))
                {
                        if (!toReturn.empty())
                                break;
                }
                else if (!inQuotes && !inLiteralQuote && inputChar == '{')
                {
                        if (toReturn.empty())
                        {
                                toReturn += inputChar;
                        }
                        else
                        {
                                theStream.putback('{');
                        }
                        break;
                }
                else if (!inQuotes && !inLiteralQuote && inputChar == '}')
                {
                        if (toReturn.empty())
                        {
                                toReturn += inputChar;
                        }
                        else
                        {
                                theStream.putback('}');
                        }
                        break;
                }
                else if (!inQuotes && !inLiteralQuote && inputChar == '=')
                {
                        if (toReturn.empty())
                        {
                                toReturn += inputChar;
                        }
                        else
                        {
                                theStream.putback('=');
                        }
                        break;
                }
                else
                {
                        toReturn += inputChar;
                }

                previousCharacter = inputChar;
        }
        return toReturn;
}

1	#include "Parser.h"
2
3	#include <filesystem>
4	#include <fstream>
5
6	#include "CommonRegexes.h"
7	#include "Log.h"
8	#include "ParserHelpers.h"
9	#include "StringUtils.h"
10
11
12
13	using std::filesystem::path;
14	using std::filesystem::u8path;
15
16
17
18	namespace commonItems
19	{
20	std::string getNextLexeme(std::istream& theStream);
21	} // namespace commonItems
22
23
24	void commonItems::absorbBOM(std::istream& theStream)	100✔
25	{
26	if (const auto firstChar = static_cast<char>(theStream.peek()); firstChar == '\xEF')	100✔
27	{
28	char bitBucket[3];
29	theStream.read(bitBucket, sizeof bitBucket);	1✔
30	}
31	}	100✔
32
33
34	void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunction& function)	7✔
35	{
36	registeredKeywordStrings.insert(std::make_pair(keyword, function));	7✔
37	}	7✔
38
39
40	void commonItems::parser::registerKeyword(const std::string& keyword, const parsingFunctionStreamOnly& function)	383✔
41	{
42	registeredKeywordStringsStreamOnly.insert(std::make_pair(keyword, function));	383✔
43	}	383✔
44
45
46	void commonItems::parser::registerRegex(const std::string& keyword, const parsingFunction& function)	381✔
47	{
48	generatedRegexes.emplace_back(std::regex(keyword), function);	381✔
49	}	381✔
50
51
52	void commonItems::parser::IgnoreUnregisteredItems()	1✔
53	{
54	registerRegex(catchallRegex, ignoreItem);	1✔
55	}	1✔
56
57
58	void commonItems::parser::IgnoreAndLogUnregisteredItems()	1✔
59	{
60	registerRegex(catchallRegex, ignoreAndLogItem);	1✔
61	}	1✔
62
63
64	void commonItems::parser::IgnoreAndStoreUnregisteredItems(std::set<std::string>& ignored_tokens)	1✔
65	{
66	registerRegex(catchallRegex, [&ignored_tokens](const std::string& key, std::istream& the_stream) {	1✔
67	ignored_tokens.insert(key);	2✔
68	ignoreItem(key, the_stream);	2✔
69	});	2✔
70	}	1✔
71
72
73	void commonItems::parser::parseStream(std::istream& theStream)	221✔
74	{
75	auto braceDepth = 0;	221✔
76	auto value = false; // tracker to indicate whether we reached the value part of key=value pair	221✔
77	std::string tokensSoFar;	221✔
78
79	while (true)
80	{
81	if (auto token = getNextToken(theStream); token)	342✔
82	{
83	tokensSoFar += *token;	187✔
84	if (*token == "=")	187✔
85	{
86	if (!value)	51✔
87	{
88	value = true; // swapping to value part.	50✔
89	continue;	50✔
90	}
91
92	// value is positive, meaning we were at value, and now we're hitting an equal. This is bad. We need to
93	// manually fast-forward to brace-lvl 0 and die.
94	char inputChar;
95	while (braceDepth)	7✔
96	{
97	theStream >> inputChar;	6✔
98	if (inputChar == '{')	6✔
99	braceDepth++;	×
100	if (inputChar == '}')	6✔
101	braceDepth--;	1✔
102	if (!isspace(inputChar))	6✔
103	tokensSoFar += inputChar;	4✔
104	}
105	Log(LogLevel::Warning) << "Broken token syntax at " << tokensSoFar;	1✔
106	return;	1✔
107	}
108	if (*token == "{")	136✔
109	braceDepth++;	66✔
110	else if (*token == "}")	70✔
111	{
112	braceDepth--;	65✔
113	if (braceDepth == 0)	65✔
114	break;	65✔
115	}
116	else
117	Log(LogLevel::Warning) << "Unknown token while parsing stream: " << *token;	5✔
118	}
119	else
120	break;	342✔
121	}	121✔
122	}	221✔
123
124
125	void commonItems::parser::parseFile(path filename)	86✔
126	{
127	std::ifstream theFile(filename);	86✔
128	if (!theFile.is_open())	86✔
129	{
130	Log(LogLevel::Error) << "Could not open " << filename << " for parsing.";	1✔
131	return;	1✔
132	}
133
134	absorbBOM(theFile);	85✔
135	parseStream(theFile);	85✔
136	theFile.close();	85✔
137	}	86✔
138
139
NEW 140	void commonItems::parser::parseFile(std::string_view filename)	×
141	{
142	#pragma warning(push)
143	#pragma warning(disable : 4996)
NEW 144	parseFile(u8path(filename));	×
145	#pragma warning(pop)
NEW 146	}	×
147
148
149	void commonItems::parser::clearRegisteredKeywords() noexcept	100✔
150	{
151	std::map<std::string, parsingFunction>().swap(registeredKeywordStrings);	100✔
152	std::map<std::string, parsingFunctionStreamOnly>().swap(registeredKeywordStringsStreamOnly);	100✔
153	std::vector<std::pair<std::regex, parsingFunction>>().swap(generatedRegexes);	100✔
154	}	100✔
155
156	void commonItems::parser::operator()(std::istream& theStream)	1✔
157	{
158	this->parseStream(theStream);	1✔
159	}	1✔
160
161	std::optional<std::string> commonItems::parser::getNextToken(std::istream& theStream)	342✔
162	{
163	theStream >> std::noskipws;	342✔
164	std::string toReturn;	342✔
165
166	auto gotToken = false;	342✔
167	while (!gotToken)	1,731✔
168	{
169	if (theStream.eof())	1,436✔
170	return {};	47✔
171
172	toReturn = getNextLexeme(theStream);	1,389✔
173
174	const auto strippedLexeme = remQuotes(toReturn);	1,389✔
175	const auto isLexemeQuoted = strippedLexeme.size() < toReturn.size();	1,389✔
176
177	auto matched = tryToMatchAgainstKeywords(toReturn, strippedLexeme, isLexemeQuoted, theStream);	1,389✔
178
179	if (!matched)	1,389✔
180	matched = tryToMatchAgainstRegexes(toReturn, strippedLexeme, isLexemeQuoted, theStream);	719✔
181
182	if (!matched)	1,389✔
183	gotToken = true;	295✔
184	}	1,389✔
185
186	if (!toReturn.empty())	295✔
187	return std::move(toReturn);	187✔
188	return std::nullopt;	108✔
189	}	342✔
190
191
192	inline bool commonItems::parser::tryToMatchAgainstKeywords(const std::string& toReturn,	1,389✔
193	const std::string& strippedLexeme,
194	bool isLexemeQuoted,
195	std::istream& theStream)
196	{
197	if (const auto& match = registeredKeywordStringsStreamOnly.find(toReturn); match != registeredKeywordStringsStreamOnly.end())	1,389✔
198	{
199	match->second(theStream);	664✔
200	return true;	664✔
201	}
202	if (const auto& match = registeredKeywordStrings.find(toReturn); match != registeredKeywordStrings.end())	725✔
203	{
204	match->second(toReturn, theStream);	5✔
205	return true;	5✔
206	}
207	if (isLexemeQuoted)	720✔
208	{
209	if (const auto& strippedMatch = registeredKeywordStringsStreamOnly.find(strippedLexeme); strippedMatch != registeredKeywordStringsStreamOnly.end())	65✔
210	{
211	strippedMatch->second(theStream);	×
212	return true;	×
213	}
214	if (const auto& strippedMatch = registeredKeywordStrings.find(strippedLexeme); strippedMatch != registeredKeywordStrings.end())	65✔
215	{
216	strippedMatch->second(toReturn, theStream);	1✔
217	return true;	1✔
218	}
219	}
220
221	return false;	719✔
222	}
223
224	inline bool commonItems::parser::tryToMatchAgainstRegexes(const std::string& toReturn,	719✔
225	const std::string& strippedLexeme,
226	bool isLexemeQuoted,
227	std::istream& theStream)
228	{
229	for (const auto& [regex, parsingFunction]: generatedRegexes)	1,408✔
230	{
231	if (std::smatch match; std::regex_match(toReturn, match, regex))	1,112✔
232	{
233	parsingFunction(toReturn, theStream);	423✔
234	return true;	423✔
235	}	1,112✔
236	}
237	if (isLexemeQuoted)	296✔
238	{
239	for (const auto& [regex, parsingFunction]: generatedRegexes)	1✔
240	{
241	if (std::smatch match; std::regex_match(strippedLexeme, match, regex))	1✔
242	{
243	parsingFunction(toReturn, theStream);	1✔
244	return true;	1✔
245	}	1✔
246	}
247	}
248	return false;	295✔
249	}
250
251
252	std::optional<std::string> commonItems::parser::getNextTokenWithoutMatching(std::istream& theStream)	1,575✔
253	{
254	theStream >> std::noskipws;	1,575✔
255
256	std::string toReturn;	1,575✔
257
258	auto gotToken = false;	1,575✔
259	while (!gotToken)	3,150✔
260	{
261	if (theStream.eof())	1,575✔
262	return std::nullopt;	×
263
264	toReturn = getNextLexeme(theStream);	1,575✔
265	gotToken = true;	1,575✔
266	}
267
268	if (!toReturn.empty())	1,575✔
269	return std::move(toReturn);	1,575✔
270	return std::nullopt;	×
271	}	1,575✔
272
273
274	std::string commonItems::getNextLexeme(std::istream& theStream)	3,330✔
275	{
276	std::string toReturn;	3,330✔
277
278	auto inQuotes = false;	3,330✔
279	auto inLiteralQuote = false;	3,330✔
280	unsigned char previousCharacter = '\0';	3,330✔
281
282	while (true)
283	{
284	char inputChar;
285	theStream >> inputChar;	28,903✔
286	if (theStream.eof())	28,903✔
287	break;	174✔
288	if (!inQuotes && inputChar == '#')	28,729✔
289	{
290	std::string bitBucket;	3✔
291	std::getline(theStream, bitBucket);	3✔
292	if (!toReturn.empty())	3✔
293	break;	×
294	}	6✔
295	else if (inputChar == '\n')	28,726✔
296	{
297	if (!inQuotes)	867✔
298	{
299	if (!toReturn.empty())	865✔
300	break;	121✔
301	}
302	else
303	{
304	// fix paradox' mistake and don't break proper names in half
305	toReturn += " ";	2✔
306	}
307	}
308	else if (inputChar == '\"' && !inQuotes && toReturn.empty())	27,859✔
309	{
310	inQuotes = true;	792✔
311	toReturn += inputChar;	792✔
312	}
313	else if (inputChar == '\"' && !inQuotes && toReturn.size() == 1 && toReturn.back() == 'R')	27,067✔
314	{
315	inLiteralQuote = true;	1✔
316	toReturn.pop_back();	1✔
317	toReturn += inputChar;	1✔
318	}
319	else if (inputChar == '(' && inLiteralQuote && toReturn.size() == 1)	27,066✔
320	{
321	continue;	1✔
322	}
323	else if (inputChar == '\"' && inLiteralQuote && previousCharacter == ')')	27,065✔
324	{
325	toReturn.pop_back();	1✔
326	toReturn += inputChar;	1✔
327	break;	1✔
328	}
329	else if (inputChar == '\"' && inQuotes && previousCharacter != '\\')	27,064✔
330	{
331	toReturn += inputChar;	792✔
332	break;	792✔
333	}
334	else if (!inQuotes && !inLiteralQuote && std::isspace(inputChar))	26,272✔
335	{
336	if (!toReturn.empty())	647✔
337	break;	268✔
338	}
339	else if (!inQuotes && !inLiteralQuote && inputChar == '{')	25,625✔
340	{
341	if (toReturn.empty())	108✔
342	{
343	toReturn += inputChar;	108✔
344	}
345	else
346	{
347	theStream.putback('{');	×
348	}
349	break;	108✔
350	}
351	else if (!inQuotes && !inLiteralQuote && inputChar == '}')	25,517✔
352	{
353	if (toReturn.empty())	90✔
354	{
355	toReturn += inputChar;	79✔
356	}
357	else
358	{
359	theStream.putback('}');	11✔
360	}
361	break;	90✔
362	}
363	else if (!inQuotes && !inLiteralQuote && inputChar == '=')	25,427✔
364	{
365	if (toReturn.empty())	1,776✔
366	{
367	toReturn += inputChar;	990✔
368	}
369	else
370	{
371	theStream.putback('=');	786✔
372	}
373	break;	1,776✔
374	}
375	else
376	{
377	toReturn += inputChar;	23,651✔
378	}
379
380	previousCharacter = inputChar;	25,572✔
381	}	25,573✔
382	return toReturn;	3,330✔
383	}	×

ParadoxGameConverters / commonItems / 12846535547

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous