feat: adding the import solver

.gitignore

@@ -4,7 +4,6 @@
 
 # Personal utilities
 warnings.log
-cformat.ps1
 
 # ArkScript
 include/Ark/Constants.hpp
@@ -28,6 +27,7 @@ afl/
 .cache/
 build/
 ninja/
+cmake-build-*/
 
 # Prerequisites
 *.d
@@ -47,10 +47,6 @@ ninja/
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
-
 # Compiled Static libraries
 *.lai
 *.la

CMakeLists.txt

@@ -122,12 +122,7 @@ target_include_directories(ArkReactor
     PUBLIC
     ${ark_SOURCE_DIR}/include)
 
-# setting up project properties
-set_target_properties(
-    ArkReactor
-    PROPERTIES
-        CXX_STANDARD 17
-        CXX_STANDARD_REQUIRED ON)
+target_compile_features(ArkReactor PRIVATE cxx_std_17)
 
 # Installation rules
 
@@ -176,6 +171,10 @@ if (ARK_BUILD_MODULES)
     add_subdirectory(${ark_SOURCE_DIR}/lib/modules)
 endif()
 
+if (ARK_BUILD_PARSER_TESTS)
+    add_subdirectory(${ark_SOURCE_DIR}/tests/parser)
+endif()
+
 if (ARK_BUILD_EXE)
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES

cmake/link_time_optimization.cmake

@@ -3,7 +3,7 @@ include(CheckIPOSupported)
 check_ipo_supported(RESULT ipo_supported)
 
 function(enable_lto target_name)
-    if (ipo_supported)
+    if (ipo_supported AND (${CMAKE_BUILD_TYPE} STREQUAL "Release"))
         if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND (CMAKE_CXX_COMPILER_VERSION MATCHES "^8\..+"))
             message(WARNING "LTO supported but not enabled to prevent https://github.com/ArkScript-lang/Ark/pull/385#issuecomment-1163597951")
         else()

include/Ark/Compiler/AST/BaseParser.hpp

@@ -74,6 +74,7 @@ namespace Ark::internal
         bool suffix(char c);
         bool number(std::string* s = nullptr);
         bool signedNumber(std::string* s = nullptr);
+        bool hexNumber(unsigned length, std::string* s = nullptr);
         bool name(std::string* s = nullptr);
         bool sequence(const std::string& s);
         bool packageName(std::string* s = nullptr);

include/Ark/Compiler/AST/Import.hpp

@@ -3,6 +3,7 @@
 
 #include <vector>
 #include <string>
+#include <numeric>
 
 #include <Ark/Platform.hpp>
 
@@ -15,25 +16,55 @@ namespace Ark::internal
          * @details Example: `(import foo.bar)` => `bar`
          *      `(import foo.bar.egg:*)` => `egg`
          *      `(import foo :a :b :c)` => `foo`
-         * 
+         *
          */
-        std::string package;
+        std::string prefix;
+
+        /**
+         * @brief Package with all the segments
+         * @details Example: `(import foo.bar)` => `{foo, bar}`
+         *      `(import foo.bar.egg:*)` => `{foo, bar, egg}`
+         *      `(import foo :a :b :c)` => `{foo}`
+         */
+        std::vector<std::string> package;
 
         /**
          * @brief Import with prefix (the package) or not
-         * 
+         *
          */
         bool with_prefix = true;
 
         /**
          * @brief List of symbols to import, can be empty if none provided
-         * 
+         *
          */
         std::vector<std::string> symbols;
 
+        inline std::string toPackageString() const
+        {
+            return std::accumulate(package.begin() + 1, package.end(), package.front(), [](const std::string& left, const std::string& right) {
+                return left + "." + right;
+            });
+        }
+
+        inline std::string packageToPath() const
+        {
+            std::size_t offset = 0;
+            if (package.front() == "std")
+                offset = 1;
+
+            return std::accumulate(
+                std::next(package.begin() + offset),
+                package.end(),
+                package[offset],
+                [](const std::string& a, const std::string& b) {
+                    return a + "/" + b;
+                });
+        }
+
         /**
          * @brief Check if we should import everything, given something like `(import foo.bar.egg:*)`
-         * 
+         *
          * @return true if all symbols of the file should be imported in the importer scope
          * @return false otherwise
          */
@@ -44,9 +75,9 @@ namespace Ark::internal
 
         /**
          * @brief Check if we should import everything with a prefix, given a `(import foo.bar.egg)`
-         * 
-         * @return true 
-         * @return false 
+         *
+         * @return true
+         * @return false
          */
         inline bool isBasic() const
         {

include/Ark/Compiler/AST/Module.hpp

@@ -0,0 +1,17 @@
+#ifndef ARK_MODULE_HPP
+#define ARK_MODULE_HPP
+
+#include <Ark/Compiler/AST/Node.hpp>
+
+namespace Ark::internal
+{
+    // TODO store something better than just the AST (AST+what we are importing as private/public/namespaced... vs all)
+    //      so that we can remember the order in which we encountered imports.
+    struct Module
+    {
+        Node ast;
+        bool has_been_processed = false;  // TODO document this
+    };
+}
+
+#endif  // ARK_MODULE_HPP

include/Ark/Compiler/AST/Parser.hpp

@@ -12,6 +12,8 @@
 #include <vector>
 #include <functional>
 
+#include <utf8_decoder.h>
+
 namespace Ark::internal
 {
     class ARK_API Parser : public BaseParser
@@ -28,6 +30,7 @@ namespace Ark::internal
     private:
         Node m_ast;
         std::vector<Import> m_imports;
+        unsigned m_allow_macro_behavior;  ///< Toggled on when inside a macro definition, off afterward
 
         void run();
 
@@ -38,7 +41,11 @@ namespace Ark::internal
         std::optional<Node> loop();
         std::optional<Node> import_();
         std::optional<Node> block();
+        std::optional<Node> functionArgs();
         std::optional<Node> function();
+        std::optional<Node> macroCondition();
+        std::optional<Node> macroBlock();
+        std::optional<Node> macroArgs();
         std::optional<Node> macro();
         std::optional<Node> functionCall();
         std::optional<Node> list();
@@ -89,6 +96,39 @@ namespace Ark::internal
                             res += '\b';
                         else if (accept(IsChar('0')))
                             res += '\0';
+                        else if (accept(IsChar('f')))
+                            res += '\f';
+                        else if (accept(IsChar('u')))
+                        {
+                            std::string seq;
+                            if (hexNumber(4, &seq))
+                            {
+                                char utf8_str[5];
+                                utf8decode(seq.c_str(), utf8_str);
+                                if (*utf8_str == '\0')
+                                    error("Invalid escape sequence", "\\u" + seq);
+                                res += utf8_str;
+                            }
+                            else
+                                error("Invalid escape sequence", "\\u");
+                        }
+                        else if (accept(IsChar('U')))
+                        {
+                            std::string seq;
+                            if (hexNumber(8, &seq))
+                            {
+                                std::size_t begin = 0;
+                                for (; seq[begin] == '0'; ++begin)
+                                    ;
+                                char utf8_str[5];
+                                utf8decode(seq.c_str() + begin, utf8_str);
+                                if (*utf8_str == '\0')
+                                    error("Invalid escape sequence", "\\U" + seq);
+                                res += utf8_str;
+                            }
+                            else
+                                error("Invalid escape sequence", "\\U");
+                        }
                         else
                         {
                             backtrack(getCount() - 1);
@@ -102,7 +142,6 @@ namespace Ark::internal
                         break;
                     else if (isEOF())
                         errorMissingSuffix('"', "string");
-                    // TODO accept(\Uxxxxx), accept(\uxxxxx)
                 }
 
                 return Node(NodeType::String, res);
@@ -121,7 +160,6 @@ namespace Ark::internal
 
             while (true)
             {
-                space();
                 if (leaf.list().size() == 1 && !accept(IsChar('.')))  // Symbol:abc
                     return std::nullopt;
 
@@ -144,6 +182,18 @@ namespace Ark::internal
             return Node(NodeType::Symbol, res);
         }
 
+        inline std::optional<Node> spread()
+        {
+            std::string res;
+            if (sequence("..."))
+            {
+                if (!name(&res))
+                    errorWithNextToken("Expected a name for the variadic");
+                return Node(NodeType::Spread, res);
+            }
+            return std::nullopt;
+        }
+
         inline std::optional<Node> nil()
         {
             if (!accept(IsChar('(')))

include/Ark/Compiler/AST/Predicates.hpp

@@ -48,6 +48,16 @@ namespace Ark::internal
         }
     } IsDigit;
 
+    inline struct IsHex : public CharPred
+    {
+        IsHex() :
+            CharPred("hex") {}
+        virtual bool operator()(const utf8_char_t::codepoint_t c) const override
+        {
+            return 0 <= c && c <= 255 && std::isxdigit(c) != 0;
+        }
+    } IsHex;
+
     inline struct IsUpper : public CharPred
     {
         IsUpper() :

include/Ark/Compiler/ImportSolver.hpp

@@ -0,0 +1,61 @@
+#ifndef ARK_COMPILER_IMPORTSOLVER_HPP
+#define ARK_COMPILER_IMPORTSOLVER_HPP
+
+#include <vector>
+#include <string>
+#include <filesystem>
+#include <unordered_map>
+
+#include <Ark/Compiler/AST/Node.hpp>
+#include <Ark/Compiler/AST/Import.hpp>
+#include <Ark/Compiler/AST/Module.hpp>
+
+namespace Ark::internal
+{
+    class ImportSolver final
+    {
+    public:
+        ImportSolver(unsigned debug, const std::vector<std::filesystem::path>& libenv);
+
+        void process(const std::filesystem::path& root, const Node& origin_ast, const std::vector<Import>& origin_imports);
+
+        const Node& ast() const noexcept;
+
+    private:
+        unsigned m_debug;
+        std::vector<std::filesystem::path> m_libenv;
+        std::filesystem::path m_root;  ///< Folder were the entry file is
+        Node m_ast;
+        std::unordered_map<std::string, Module> m_modules;  ///< Package to module map
+        // TODO is this ok? is this fine? this is sort of ugly
+        std::vector<std::string> m_imported;  ///< List of imports, in the order they were found and parsed
+
+        /**
+         * @brief Visits the AST, looking for import nodes to replace with their parsed module version
+         * @param ast
+         * @return
+         */
+        std::pair<Node, bool> findAndReplaceImports(const Node& ast);
+
+        /**
+         * @brief Parse a given file and returns a list of its imports.
+         *        The AST is parsed and stored in m_modules[import.prefix]
+         * 
+         * @param file path to the file containing the import
+         * @param import current import directive
+         * @return std::vector<Import> imports found in the processed file
+         */
+        std::vector<Import> parseImport(const std::filesystem::path& file, const Import& import);
+
+        /**
+         * @brief Search for an import file, using the root file path
+         * 
+         * @param file path to the file containing the import
+         * @param import current import directive
+         * @return std::filesystem::path 
+         */
+        std::filesystem::path findFile(const std::filesystem::path& file, const Import& import);
+    };
+}
+
+#endif

include/Ark/Compiler/JsonCompiler.hpp

@@ -3,6 +3,7 @@
 
 #include <vector>
 #include <string>
+#include <filesystem>
 
 #include <Ark/Constants.hpp>
 #include <Ark/Platform.hpp>
@@ -19,7 +20,7 @@ namespace Ark
          *
          * @param debug the debug level
          */
-        JsonCompiler(unsigned debug, const std::vector<std::string>& libenv);
+        JsonCompiler(unsigned debug, const std::vector<std::filesystem::path>& libenv);
 
         /**
          * @brief Feed the differents variables with information taken from the given source code file