14956639333

Committed 11 May 2025 02:13PM UTC coverage: 86.495% (+0.02%) from 86.474%

Build # 14956639333

Build Type

Pull #535

github

Committed by

web-flow

Commit Message

Merge fa81c887e into 3f1c6e9e3

Pull Request Pull Request #535: Feat/parsing errors context

Run Details

208 of 230 new or added lines in 10 files covered. (90.43%)

3 existing lines in 2 files now uncovered.

6949 of 8034 relevant lines covered (86.49%)

79310.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.28

/src/arkreactor/Compiler/Package/ImportSolver.cpp

#include <Ark/Compiler/Package/ImportSolver.hpp>

#include <ranges>
#include <algorithm>
#include <fmt/core.h>

#include <Ark/Files.hpp>
#include <Ark/Exceptions.hpp>
#include <Ark/Compiler/AST/Parser.hpp>

namespace Ark::internal
{
    ImportSolver::ImportSolver(const unsigned debug, const std::vector<std::filesystem::path>& libenv) :
        Pass("ImportSolver", debug), m_debug_level(debug), m_libenv(libenv), m_ast()
    {}

    ImportSolver& ImportSolver::setup(const std::filesystem::path& root, const std::vector<Import>& origin_imports)
    {
        if (is_directory(root))
            m_root = root;
        else
            m_root = root.parent_path();

        for (const auto& origin_import : std::ranges::reverse_view(origin_imports))
            m_imports.push(origin_import);

        return *this;
    }

    void ImportSolver::process(const Node& origin_ast)
    {
        m_logger.traceStart("process");

        while (!m_imports.empty())
        {
            Import import = m_imports.top();
            m_logger.debug("Importing {}", import.toPackageString());

            // Remove the top element to process the other imports
            // It needs to be removed first because we might be adding
            // other imports later and don't want to pop THEM
            m_imports.pop();
            const auto package = import.toPackageString();

            if (m_packages.contains(package))
            {
                // merge the definition, so that we can generate valid Full Qualified Names in the name & scope resolver
                m_packages[package].import.with_prefix |= import.with_prefix;
                m_packages[package].import.is_glob |= import.is_glob;
                for (auto&& symbol : import.symbols)
                    m_packages[package].import.symbols.push_back(symbol);
            }
            else
            {
                // NOTE: since the "file" (=root) argument doesn't change between all calls, we could get rid of it
                std::vector<Import> temp = parseImport(m_root, import);
                for (auto& additional_import : std::ranges::reverse_view(temp))
                    m_imports.push(additional_import);
            }
        }

        m_logger.traceStart("findAndReplaceImports");
        m_ast = findAndReplaceImports(origin_ast).first;
        m_logger.traceEnd();

        m_logger.traceEnd();
    }

    std::pair<Node, bool> ImportSolver::findAndReplaceImports(const Node& ast)
    {
        Node x = ast;
        if (x.nodeType() == NodeType::List)
        {
            if (x.constList().size() >= 2 && x.constList()[0].nodeType() == NodeType::Keyword &&
                x.constList()[0].keyword() == Keyword::Import)
            {
                // compute the package string: foo.bar.egg
                const auto import_node = x.constList()[1].constList();
                const std::string package = std::accumulate(
                    std::next(import_node.begin()),
                    import_node.end(),
                    import_node[0].string(),
                    [](const std::string& acc, const Node& elem) -> std::string {
                        return acc + "." + elem.string();
                    });

                // if it wasn't imported already, register it
                if (std::ranges::find(m_imported, package) == m_imported.end())
                {
                    m_imported.push_back(package);
                    // modules are already handled, we can safely replace the node
                    x = m_packages[package].ast;
                    if (!m_packages[package].has_been_processed)
                    {
                        const auto import = m_packages[package].import;

                        // prefix to lowercase ; usually considered unsafe (https://devblogs.microsoft.com/oldnewthing/20241007-00/?p=110345)
                        // but we are dealing with prefix from filenames, thus we can somewhat assume we are in safe zone
                        std::string prefix = import.prefix;
                        std::ranges::transform(
                            prefix, prefix.begin(),
                            [](auto c) {
                                return std::tolower(c);
                            });

                        x = Node(Namespace {
                            .name = prefix,
                            .is_glob = import.is_glob,
                            .with_prefix = import.with_prefix,
                            .symbols = import.symbols,
                            .ast = std::make_shared<Node>(findAndReplaceImports(x).first) });
                        x.arkNamespace().ast->setPos(ast.line(), ast.col());
                        x.arkNamespace().ast->setFilename(ast.filename());
                    }
                    // we parsed an import node, return true in the pair to notify the caller
                    return std::make_pair(x, /* is_import= */ true);
                }

                // Replace by empty node to avoid breaking the code gen
                x = Node(NodeType::List);
                x.push_back(Node(Keyword::Begin));
            }
            else
            {
                for (std::size_t i = 0; i < x.constList().size(); ++i)
                {
                    auto [node, is_import] = findAndReplaceImports(x.constList()[i]);
                    x.list()[i] = node;
                }
            }
        }

        return std::make_pair(x, /* is_import= */ false);
    }

    const Node& ImportSolver::ast() const noexcept
    {
        return m_ast;
    }

    std::vector<Import> ImportSolver::parseImport(const std::filesystem::path& base_path, const Import& import)
    {
        m_logger.traceStart(fmt::format("parseImport {}", base_path.string()));

        const auto path = findFile(base_path, import);
        if (path.extension() == ".arkm")  // Nothing to import in case of modules
        {
            // Creating an import node that will stay there when visiting the AST and
            // replacing the imports with their parsed module
            auto module_node = Node(NodeType::List);
            module_node.push_back(Node(Keyword::Import));

            auto package_node = Node(NodeType::List);
            std::ranges::transform(import.package, std::back_inserter(package_node.list()), [](const std::string& stem) {
                return Node(NodeType::String, stem);
            });
            module_node.push_back(package_node);
            // empty symbols list
            module_node.push_back(Node(NodeType::List));

            m_packages[import.toPackageString()] = Package {
                module_node,
                import,
                true
            };

            return {};
        }

        Parser parser(m_debug_level);
        const std::string code = Utils::readFile(path.generic_string());
        parser.process(path.string(), code);
        m_packages[import.toPackageString()] = Package {
            parser.ast(),
            import,
            false
        };

        m_logger.traceEnd();
        return parser.imports();
    }

    std::optional<std::filesystem::path> testExtensions(const std::filesystem::path& folder, const std::string& package_path)
    {
        if (auto code_path = folder / (package_path + ".ark"); std::filesystem::exists(code_path))
            return code_path;
        if (auto module_path = folder / (package_path + ".arkm"); std::filesystem::exists(module_path))
            return module_path;
        return {};
    }

    std::filesystem::path ImportSolver::findFile(const std::filesystem::path& file, const Import& import) const
    {
        const std::string package_path = import.packageToPath();
        if (auto maybe_path = testExtensions(m_root, package_path); maybe_path.has_value())
            return maybe_path.value();

        // search in all folders in environment path
        for (const auto& path : m_libenv)
        {
            if (auto maybe_path = testExtensions(path, package_path); maybe_path.has_value())
                return maybe_path.value();
        }

        // fallback, we couldn't find the file
        throw CodeError(
            fmt::format("While processing file {}, couldn't import {}: file not found",
                        file.generic_string(), import.toPackageString()),
            CodeErrorContext(
                file.generic_string(),
                import.line,
                import.col,
                fmt::format("(import {})", import.toPackageString())));
    }
}

1	#include <Ark/Compiler/Package/ImportSolver.hpp>
2
3	#include <ranges>
4	#include <algorithm>
5	#include <fmt/core.h>
6
7	#include <Ark/Files.hpp>
8	#include <Ark/Exceptions.hpp>
9	#include <Ark/Compiler/AST/Parser.hpp>
10
11	namespace Ark::internal
12	{
13	ImportSolver::ImportSolver(const unsigned debug, const std::vector<std::filesystem::path>& libenv) :	813✔
14	Pass("ImportSolver", debug), m_debug_level(debug), m_libenv(libenv), m_ast()	271✔
15	{}	813✔
16
17	ImportSolver& ImportSolver::setup(const std::filesystem::path& root, const std::vector<Import>& origin_imports)	241✔
18	{	241✔
19	if (is_directory(root))	241✔
20	m_root = root;	4✔
21	else
22	m_root = root.parent_path();	237✔
23
24	for (const auto& origin_import : std::ranges::reverse_view(origin_imports))	308✔
25	m_imports.push(origin_import);	67✔
26
27	return *this;	241✔
28	}
29
30	void ImportSolver::process(const Node& origin_ast)	241✔
31	{	241✔
32	m_logger.traceStart("process");	241✔
33
34	while (!m_imports.empty())	360✔
35	{
36	Import import = m_imports.top();	119✔
37	m_logger.debug("Importing {}", import.toPackageString());	119✔
38
39	// Remove the top element to process the other imports
40	// It needs to be removed first because we might be adding
41	// other imports later and don't want to pop THEM
42	m_imports.pop();	119✔
43	const auto package = import.toPackageString();	119✔
44
45	if (m_packages.contains(package))	119✔
46	{
47	// merge the definition, so that we can generate valid Full Qualified Names in the name & scope resolver
48	m_packages[package].import.with_prefix \|= import.with_prefix;	25✔
49	m_packages[package].import.is_glob \|= import.is_glob;	25✔
50	for (auto&& symbol : import.symbols)	30✔
51	m_packages[package].import.symbols.push_back(symbol);	5✔
52	}	25✔
53	else
54	{
55	// NOTE: since the "file" (=root) argument doesn't change between all calls, we could get rid of it
56	std::vector<Import> temp = parseImport(m_root, import);	94✔
57	for (auto& additional_import : std::ranges::reverse_view(temp))	146✔
58	m_imports.push(additional_import);	52✔
59	}	94✔
60	}	119✔
61
62	m_logger.traceStart("findAndReplaceImports");	241✔
63	m_ast = findAndReplaceImports(origin_ast).first;	241✔
64	m_logger.traceEnd();	241✔
65
66	m_logger.traceEnd();	241✔
67	}	241✔
68
69	std::pair<Node, bool> ImportSolver::findAndReplaceImports(const Node& ast)	115,463✔
70	{	115,463✔
71	Node x = ast;	115,463✔
72	if (x.nodeType() == NodeType::List)	115,463✔
73	{
74	if (x.constList().size() >= 2 && x.constList()[0].nodeType() == NodeType::Keyword &&	24,223✔
75	x.constList()[0].keyword() == Keyword::Import)	7,480✔
76	{
77	// compute the package string: foo.bar.egg
78	const auto import_node = x.constList()[1].constList();	119✔
79	const std::string package = std::accumulate(	238✔
80	std::next(import_node.begin()),	119✔
81	import_node.end(),	119✔
82	import_node[0].string(),	119✔
83	[](const std::string& acc, const Node& elem) -> std::string {	86✔
84	return acc + "." + elem.string();	86✔
85	});	×
86
87	// if it wasn't imported already, register it
88	if (std::ranges::find(m_imported, package) == m_imported.end())	119✔
89	{
90	m_imported.push_back(package);	94✔
91	// modules are already handled, we can safely replace the node
92	x = m_packages[package].ast;	94✔
93	if (!m_packages[package].has_been_processed)	94✔
94	{
95	const auto import = m_packages[package].import;	93✔
96
97	// prefix to lowercase ; usually considered unsafe (https://devblogs.microsoft.com/oldnewthing/20241007-00/?p=110345)
98	// but we are dealing with prefix from filenames, thus we can somewhat assume we are in safe zone
99	std::string prefix = import.prefix;	93✔
100	std::ranges::transform(	93✔
101	prefix, prefix.begin(),	93✔
102	[](auto c) {	494✔
103	return std::tolower(c);	494✔
104	});
105
106	x = Node(Namespace {	372✔
107	.name = prefix,	93✔
108	.is_glob = import.is_glob,	93✔
109	.with_prefix = import.with_prefix,	93✔
110	.symbols = import.symbols,	93✔
111	.ast = std::make_shared<Node>(findAndReplaceImports(x).first) });	93✔
112	x.arkNamespace().ast->setPos(ast.line(), ast.col());	93✔
113	x.arkNamespace().ast->setFilename(ast.filename());	93✔
114	}	93✔
115	// we parsed an import node, return true in the pair to notify the caller
116	return std::make_pair(x, /* is_import= */ true);	94✔
117	}
118
119	// Replace by empty node to avoid breaking the code gen
120	x = Node(NodeType::List);	25✔
121	x.push_back(Node(Keyword::Begin));	25✔
122	}	119✔
123	else
124	{
125	for (std::size_t i = 0; i < x.constList().size(); ++i)	131,753✔
126	{
127	auto [node, is_import] = findAndReplaceImports(x.constList()[i]);	115,129✔
128	x.list()[i] = node;	115,129✔
129	}	115,129✔
130	}
131	}	16,649✔
132
133	return std::make_pair(x, /* is_import= */ false);	115,369✔
134	}	115,463✔
135
136	const Node& ImportSolver::ast() const noexcept	241✔
137	{	241✔
138	return m_ast;	241✔
139	}
140
141	std::vector<Import> ImportSolver::parseImport(const std::filesystem::path& base_path, const Import& import)	94✔
142	{	94✔
143	m_logger.traceStart(fmt::format("parseImport {}", base_path.string()));	94✔
144
145	const auto path = findFile(base_path, import);	94✔
146	if (path.extension() == ".arkm") // Nothing to import in case of modules	94✔
147	{
148	// Creating an import node that will stay there when visiting the AST and
149	// replacing the imports with their parsed module
150	auto module_node = Node(NodeType::List);	1✔
151	module_node.push_back(Node(Keyword::Import));	1✔
152
153	auto package_node = Node(NodeType::List);	1✔
154	std::ranges::transform(import.package, std::back_inserter(package_node.list()), [](const std::string& stem) {	2✔
155	return Node(NodeType::String, stem);	1✔
156	});
157	module_node.push_back(package_node);	1✔
158	// empty symbols list
159	module_node.push_back(Node(NodeType::List));	1✔
160
161	m_packages[import.toPackageString()] = Package {	2✔
162	module_node,	1✔
163	import,	1✔
164	true
165	};
166
167	return {};	1✔
168	}	1✔
169
170	Parser parser(m_debug_level);	93✔
171	const std::string code = Utils::readFile(path.generic_string());	93✔
172	parser.process(path.string(), code);	93✔
173	m_packages[import.toPackageString()] = Package {	279✔
174	parser.ast(),	93✔
175	import,	93✔
176	false
177	};
178
179	m_logger.traceEnd();	93✔
180	return parser.imports();	93✔
181	}	94✔
182
183	std::optional<std::filesystem::path> testExtensions(const std::filesystem::path& folder, const std::string& package_path)	152✔
184	{	152✔
185	if (auto code_path = folder / (package_path + ".ark"); std::filesystem::exists(code_path))	245✔
186	return code_path;	93✔
187	if (auto module_path = folder / (package_path + ".arkm"); std::filesystem::exists(module_path))	60✔
188	return module_path;	1✔
189	return {};	58✔
190	}	152✔
191
192	std::filesystem::path ImportSolver::findFile(const std::filesystem::path& file, const Import& import) const	94✔
193	{	94✔
194	const std::string package_path = import.packageToPath();	94✔
195	if (auto maybe_path = testExtensions(m_root, package_path); maybe_path.has_value())	130✔
196	return maybe_path.value();	36✔
197
198	// search in all folders in environment path
199	for (const auto& path : m_libenv)	58✔
200	{
201	if (auto maybe_path = testExtensions(path, package_path); maybe_path.has_value())	116✔
202	return maybe_path.value();	58✔
203	}	58✔
204
205	// fallback, we couldn't find the file
206	throw CodeError(	×
207	fmt::format("While processing file {}, couldn't import {}: file not found",	×
208	file.generic_string(), import.toPackageString()),	×
NEW 209	CodeErrorContext(	×
NEW 210	file.generic_string(),	×
NEW 211	import.line,	×
NEW 212	import.col,	×
NEW 213	fmt::format("(import {})", import.toPackageString())));	×
214	}	94✔
215	}

ArkScript-lang / Ark / 14956639333

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous