11629611787

Committed 01 Nov 2024 12:48PM UTC coverage: 77.319% (+0.3%) from 77.042%

Build # 11629611787

Build Type

push

github

Committed by

SuperFola

Commit Message

feat(tests): adding first test for IR generation and optimization

Run Details

5209 of 6737 relevant lines covered (77.32%)

9473.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.84

/src/arkreactor/Compiler/Package/ImportSolver.cpp

#include <Ark/Compiler/Package/ImportSolver.hpp>

#include <ranges>
#include <algorithm>
#include <Ark/Exceptions.hpp>
#include <fmt/core.h>

#include <Ark/Files.hpp>
#include <Ark/Compiler/AST/Parser.hpp>

namespace Ark::internal
{
    ImportSolver::ImportSolver(const unsigned debug, const std::vector<std::filesystem::path>& libenv) :
        Pass("ImportSolver", debug), m_debug_level(debug), m_libenv(libenv), m_ast()
    {}

    ImportSolver& ImportSolver::setup(const std::filesystem::path& root, const std::vector<Import>& origin_imports)
    {
        if (is_directory(root))
            m_root = root;
        else
            m_root = root.parent_path();

        for (const auto& origin_import : std::ranges::reverse_view(origin_imports))
            m_imports.push(origin_import);

        return *this;
    }

    void ImportSolver::process(const Node& origin_ast)
    {
        m_logger.traceStart("process");

        while (!m_imports.empty())
        {
            Import import = m_imports.top();
            m_logger.debug("Importing {}", import.toPackageString());

            // Remove the top element to process the other imports
            // It needs to be removed first because we might be adding
            // other imports later and don't want to pop THEM
            m_imports.pop();

            // TODO: add special handling for each type of import (prefixed, with symbols, glob pattern)
            if (!m_modules.contains(import.toPackageString()))
            {
                // NOTE: since the "file" (=root) argument doesn't change between all calls, we could get rid of it
                std::vector<Import> additional_imports = parseImport(m_root, import);
                // TODO import and store the new node as a Module node.
                //      Module nodes should be scoped relatively to their packages
                //      They should provide specific methods to resolve symbols,
                //      mark them as public or private.
                //      OR we could have a map<import, module>, update the module
                //      accordingly, and once we are done concat all the nodes
                //      in a single AST.
                for (auto& additional_import : std::ranges::reverse_view(additional_imports))
                    m_imports.push(additional_import);
            }
            else
            {
                // TODO: if we already imported a package we should merge their definition
                //          (import foo:*) > (import foo:a)  -- no prefix
                //          (import foo)  -- with prefix
                //          and then decide what to do with the module
            }
        }

        m_logger.traceStart("findAndReplaceImports");
        m_ast = findAndReplaceImports(origin_ast).first;
        m_logger.traceEnd();

        m_logger.traceEnd();
    }

    std::pair<Node, bool> ImportSolver::findAndReplaceImports(const Node& ast)
    {
        Node x = ast;
        if (x.nodeType() == NodeType::List)
        {
            if (x.constList().size() >= 2 && x.constList()[0].nodeType() == NodeType::Keyword &&
                x.constList()[0].keyword() == Keyword::Import)
            {
                // TODO maybe we'll have problems with :* ?
                std::string package = std::accumulate(
                    std::next(x.constList()[1].constList().begin()),
                    x.constList()[1].constList().end(),
                    x.constList()[1].constList()[0].string(),
                    [](const std::string& acc, const Node& elem) -> std::string {
                        return acc + "." + elem.string();
                    });

                if (std::ranges::find(m_imported, package) == m_imported.end())
                {
                    m_imported.push_back(package);
                    // modules are already handled, we can safely replace the node
                    x = m_modules[package].ast;
                    if (!m_modules[package].has_been_processed)
                        x = findAndReplaceImports(x).first;  // FIXME?
                    return std::make_pair(x, !m_modules[package].has_been_processed);
                }

                // Replace by empty node to avoid breaking the code gen
                x = Node(NodeType::List);
                x.push_back(Node(Keyword::Begin));
            }
            else
            {
                for (std::size_t i = 0; i < x.constList().size(); ++i)
                {
                    auto [node, is_import] = findAndReplaceImports(x.constList()[i]);
                    if (!is_import)
                        x.list()[i] = node;
                    else
                    {
                        if (node.constList().size() > 1)
                        {
                            x.list()[i] = node.constList()[1];
                            // NOTE maybe maybe maybe
                            // why do we start at 2 and not 1?
                            for (std::size_t j = 2, end_j = node.constList().size(); j < end_j; ++j)
                            {
                                if (i + j - 1 < x.list().size())
                                    x.list().insert(
                                        x.list().begin() + static_cast<std::vector<Node>::difference_type>(i + j - 1),
                                        node.constList()[j]);
                                else
                                    x.list().push_back(node.constList()[j]);
                            }

                            // -2 because we skipped the Begin node and the first node of the block isn't inserted
                            // but replaces an existing one
                            i += node.constList().size() - 2;
                        }
                        else
                            x.list()[i] = node;
                    }
                }
            }
        }

        return std::make_pair(x, false);
    }

    const Node& ImportSolver::ast() const noexcept
    {
        return m_ast;
    }

    std::vector<Import> ImportSolver::parseImport(const std::filesystem::path& base_path, const Import& import)
    {
        m_logger.traceStart(fmt::format("parseImport {}", base_path.string()));

        const auto path = findFile(base_path, import);
        if (path.extension() == ".arkm")  // Nothing to import in case of modules
        {
            // Creating an import node that will stay there when visiting the AST and
            // replacing the imports with their parsed module
            auto module_node = Node(NodeType::List);
            module_node.push_back(Node(Keyword::Import));

            auto package_node = Node(NodeType::List);
            std::ranges::transform(import.package, std::back_inserter(package_node.list()), [](const std::string& stem) {
                return Node(NodeType::String, stem);
            });
            module_node.push_back(package_node);
            // empty symbols list
            module_node.push_back(Node(NodeType::List));

            m_modules[import.toPackageString()] = Module {
                module_node,
                true
            };

            return {};
        }

        Parser parser(m_debug_level);
        const std::string code = Utils::readFile(path.generic_string());
        parser.process(path.string(), code);
        m_modules[import.toPackageString()] = Module {
            parser.ast(),
            false
        };

        m_logger.traceEnd();
        return parser.imports();
    }

    std::optional<std::filesystem::path> testExtensions(const std::filesystem::path& folder, const std::string& package_path)
    {
        if (auto code_path = folder / (package_path + ".ark"); std::filesystem::exists(code_path))
            return code_path;
        if (auto module_path = folder / (package_path + ".arkm"); std::filesystem::exists(module_path))
            return module_path;
        return {};
    }

    std::filesystem::path ImportSolver::findFile(const std::filesystem::path& file, const Import& import) const
    {
        const std::string package_path = import.packageToPath();
        if (auto maybe_path = testExtensions(m_root, package_path); maybe_path.has_value())
            return maybe_path.value();

        // search in all folders in environment path
        for (const auto& path : m_libenv)
        {
            if (auto maybe_path = testExtensions(path, package_path); maybe_path.has_value())
                return maybe_path.value();
        }

        // fallback, we couldn't find the file
        throw CodeError(
            fmt::format("While processing file {}, couldn't import {}: file not found",
                        file.generic_string(), import.toPackageString()),
            file.generic_string(),
            import.line,
            import.col,
            fmt::format("(import {})", import.toPackageString()));
    }
}

1	#include <Ark/Compiler/Package/ImportSolver.hpp>
2
3	#include <ranges>
4	#include <algorithm>
5	#include <Ark/Exceptions.hpp>
6	#include <fmt/core.h>
7
8	#include <Ark/Files.hpp>
9	#include <Ark/Compiler/AST/Parser.hpp>
10
11	namespace Ark::internal
12	{
13	ImportSolver::ImportSolver(const unsigned debug, const std::vector<std::filesystem::path>& libenv) :	300✔
14	Pass("ImportSolver", debug), m_debug_level(debug), m_libenv(libenv), m_ast()	100✔
15	{}	300✔
16
17	ImportSolver& ImportSolver::setup(const std::filesystem::path& root, const std::vector<Import>& origin_imports)	72✔
18	{	72✔
19	if (is_directory(root))	72✔
20	m_root = root;	4✔
21	else
22	m_root = root.parent_path();	68✔
23
24	for (const auto& origin_import : std::ranges::reverse_view(origin_imports))	79✔
25	m_imports.push(origin_import);	7✔
26
27	return *this;	72✔
28	}
29
30	void ImportSolver::process(const Node& origin_ast)	72✔
31	{	72✔
32	m_logger.traceStart("process");	72✔
33
34	while (!m_imports.empty())	89✔
35	{
36	Import import = m_imports.top();	17✔
37	m_logger.debug("Importing {}", import.toPackageString());	17✔
38
39	// Remove the top element to process the other imports
40	// It needs to be removed first because we might be adding	×
41	// other imports later and don't want to pop THEM
42	m_imports.pop();	17✔
43
44	// TODO: add special handling for each type of import (prefixed, with symbols, glob pattern)
45	if (!m_modules.contains(import.toPackageString()))	17✔
46	{
47	// NOTE: since the "file" (=root) argument doesn't change between all calls, we could get rid of it
48	std::vector<Import> additional_imports = parseImport(m_root, import);	11✔
49	// TODO import and store the new node as a Module node.
50	// Module nodes should be scoped relatively to their packages	×
51	// They should provide specific methods to resolve symbols,
52	// mark them as public or private.	×
53	// OR we could have a map<import, module>, update the module
54	// accordingly, and once we are done concat all the nodes
55	// in a single AST.
56	for (auto& additional_import : std::ranges::reverse_view(additional_imports))	21✔
57	m_imports.push(additional_import);	10✔
58	}	11✔
59	else
60	{	×
61	// TODO: if we already imported a package we should merge their definition
62	// (import foo:*) > (import foo:a) -- no prefix	×
63	// (import foo) -- with prefix
64	// and then decide what to do with the module
65	}
66	}	17✔
67
68	m_logger.traceStart("findAndReplaceImports");	72✔
69	m_ast = findAndReplaceImports(origin_ast).first;	72✔
70	m_logger.traceEnd();	72✔
71
72	m_logger.traceEnd();	72✔
73	}	72✔
74
75	std::pair<Node, bool> ImportSolver::findAndReplaceImports(const Node& ast)	6,250✔
76	{	6,250✔
77	Node x = ast;	6,250✔
78	if (x.nodeType() == NodeType::List)	6,250✔
79	{
80	if (x.constList().size() >= 2 && x.constList()[0].nodeType() == NodeType::Keyword &&	2,762✔
81	x.constList()[0].keyword() == Keyword::Import)	740✔
82	{
83	// TODO maybe we'll have problems with :* ?
84	std::string package = std::accumulate(	34✔
85	std::next(x.constList()[1].constList().begin()),	17✔
86	x.constList()[1].constList().end(),	17✔
87	x.constList()[1].constList()[0].string(),	17✔
88	[](const std::string& acc, const Node& elem) -> std::string {	10✔
89	return acc + "." + elem.string();	10✔
90	});	×
91
92	if (std::ranges::find(m_imported, package) == m_imported.end())	17✔
93	{
94	m_imported.push_back(package);	11✔
95	// modules are already handled, we can safely replace the node
96	x = m_modules[package].ast;	11✔
97	if (!m_modules[package].has_been_processed)	11✔
98	x = findAndReplaceImports(x).first; // FIXME?	11✔
99	return std::make_pair(x, !m_modules[package].has_been_processed);	11✔
100	}
101
102	// Replace by empty node to avoid breaking the code gen
103	x = Node(NodeType::List);	6✔
104	x.push_back(Node(Keyword::Begin));	6✔
105	}	17✔
106	else
107	{
108	for (std::size_t i = 0; i < x.constList().size(); ++i)	8,172✔
109	{
110	auto [node, is_import] = findAndReplaceImports(x.constList()[i]);	12,533✔
111	if (!is_import)	6,167✔
112	x.list()[i] = node;	6,156✔
113	else
114	{
115	if (node.constList().size() > 1)	11✔
116	{
117	x.list()[i] = node.constList()[1];	11✔
118	// NOTE maybe maybe maybe
119	// why do we start at 2 and not 1?
120	for (std::size_t j = 2, end_j = node.constList().size(); j < end_j; ++j)	188✔
121	{
122	if (i + j - 1 < x.list().size())	166✔
123	x.list().insert(	244✔
124	x.list().begin() + static_cast<std::vector<Node>::difference_type>(i + j - 1),	122✔
125	node.constList()[j]);	122✔
126	else
127	x.list().push_back(node.constList()[j]);	44✔
128	}	166✔
129
130	// -2 because we skipped the Begin node and the first node of the block isn't inserted
131	// but replaces an existing one
132	i += node.constList().size() - 2;	11✔
133	}	11✔
134	else
135	x.list()[i] = node;	×
136	}
137	}	6,167✔
138	}
139	}	2,011✔
140
141	return std::make_pair(x, false);	6,239✔
142	}	6,250✔
143
144	const Node& ImportSolver::ast() const noexcept	72✔
145	{	72✔
146	return m_ast;	72✔
147	}
148
149	std::vector<Import> ImportSolver::parseImport(const std::filesystem::path& base_path, const Import& import)	11✔
150	{	11✔
151	m_logger.traceStart(fmt::format("parseImport {}", base_path.string()));	11✔
152
153	const auto path = findFile(base_path, import);	11✔
154	if (path.extension() == ".arkm") // Nothing to import in case of modules	11✔
155	{
156	// Creating an import node that will stay there when visiting the AST and
157	// replacing the imports with their parsed module	×
158	auto module_node = Node(NodeType::List);	×
159	module_node.push_back(Node(Keyword::Import));	1✔
160
161	auto package_node = Node(NodeType::List);	×
162	std::ranges::transform(import.package, std::back_inserter(package_node.list()), [](const std::string& stem) {	×
163	return Node(NodeType::String, stem);	×
164	});
165	module_node.push_back(package_node);	×
166	// empty symbols list
167	module_node.push_back(Node(NodeType::List));	×
168
169	m_modules[import.toPackageString()] = Module {	×
170	module_node,	×
171	true
172	};
173
174	return {};	×
175	}	×
176
177	Parser parser(m_debug_level);	11✔
178	const std::string code = Utils::readFile(path.generic_string());	11✔
179	parser.process(path.string(), code);	11✔
180	m_modules[import.toPackageString()] = Module {	22✔
181	parser.ast(),	11✔
182	false
183	};
184
185	m_logger.traceEnd();	11✔
186	return parser.imports();	11✔
187	}	11✔
188
189	std::optional<std::filesystem::path> testExtensions(const std::filesystem::path& folder, const std::string& package_path)	15✔
190	{	15✔
191	if (auto code_path = folder / (package_path + ".ark"); std::filesystem::exists(code_path))	26✔
192	return code_path;	11✔
193	if (auto module_path = folder / (package_path + ".arkm"); std::filesystem::exists(module_path))	4✔
194	return module_path;	×
195	return {};	4✔
196	}	15✔
197
198	std::filesystem::path ImportSolver::findFile(const std::filesystem::path& file, const Import& import) const	11✔
199	{	11✔
200	const std::string package_path = import.packageToPath();	11✔
201	if (auto maybe_path = testExtensions(m_root, package_path); maybe_path.has_value())	18✔
202	return maybe_path.value();	7✔
203
204	// search in all folders in environment path
205	for (const auto& path : m_libenv)	4✔
206	{
207	if (auto maybe_path = testExtensions(path, package_path); maybe_path.has_value())	8✔
208	return maybe_path.value();	4✔
209	}	4✔
210
211	// fallback, we couldn't find the file
212	throw CodeError(	×
213	fmt::format("While processing file {}, couldn't import {}: file not found",	×
214	file.generic_string(), import.toPackageString()),	×
215	file.generic_string(),	×
216	import.line,	×
217	import.col,	×
218	fmt::format("(import {})", import.toPackageString()));	×
219	}	11✔
220	}

ArkScript-lang / Ark / 11629611787

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous