Merge pull request #486 from ArkScript-lang/feat/compiler-passes

Feat/compiler passes

CHANGELOG.md

@@ -20,6 +20,8 @@
 - warning when the formatter deletes comment(s) by mistake
 - check on arguments passed to `list`, `concat`, `append` and friends to only push valid nodes (that produces a value)
 - `$paste` to paste a node inside a maro without evaluating it further ; useful to stop recursive evaluation of nodes inside function macros
+- introduced `Ark::internal::Pass` to describe compiler passes: they all output an AST (parser, import solver, macro processor, and optimizer for now)
+- add `-f(no-)importsolver`, `-f(no-)macroprocessor` and `-f(no-)optimizer` to toggle on and off those compiler passes
 
 ### Changed
 - instructions are on 4 bytes: 1 byte for the instruction, 1 byte of padding, 2 bytes for an immediate argument
@@ -64,6 +66,7 @@
 - checking for invalid symbols when defining a function through a macro
 - added a max macro unification depth
 - added a max macro evaluation depth
+- introduced `internal::listInstructions` with the different instructions, to be used by the compiler and name resolution pass
 
 ### Removed
 - removed unused `NodeType::Closure`

include/Ark/Compiler/AST/Optimizer.hpp

@@ -2,7 +2,7 @@
  * @file Optimizer.hpp
  * @author Alexandre Plateau ([email protected])
  * @brief Optimizes a given ArkScript AST
- * @version 1.0
+ * @version 1.1
  * @date 2024-07-09
  *
  * @copyright Copyright (c) 2020-2024
@@ -17,6 +17,7 @@
 #include <string>
 #include <cinttypes>
 
+#include <Ark/Compiler/Pass.hpp>
 #include <Ark/Compiler/AST/Node.hpp>
 #include <Ark/Exceptions.hpp>
 
@@ -26,7 +27,7 @@ namespace Ark::internal
      * @brief The ArkScript AST optimizer
      *
      */
-    class Optimizer
+    class Optimizer final : public Pass
     {
     public:
         /**
@@ -41,18 +42,17 @@ namespace Ark::internal
          *
          * @param ast
          */
-        void process(const Node& ast);
+        void process(const Node& ast) override;
 
         /**
          * @brief Returns the modified AST
          *
          * @return const Node&
          */
-        [[nodiscard]] const Node& ast() const noexcept;
+        [[nodiscard]] const Node& ast() const noexcept override;
 
     private:
         Node m_ast;
-        unsigned m_debug;
         std::unordered_map<std::string, unsigned> m_sym_appearances;
 
         /**
@@ -67,7 +67,7 @@ namespace Ark::internal
          * @brief Iterate over the AST and remove unused top level functions and constants
          *
          */
-        void remove_unused();
+        void removeUnused();
 
         /**
          * @brief Run a given functor on the global scope symbols

include/Ark/Compiler/AST/Parser.hpp

@@ -2,7 +2,7 @@
  * @file Parser.hpp
  * @author Alexandre Plateau ([email protected])
  * @brief Parse ArkScript code, but do not handle any import declarations
- * @version 0.1
+ * @version 0.2
  * @date 2024-05-12
  *
  * @copyright Copyright (c) 2024
@@ -15,6 +15,7 @@
 #include <Ark/Compiler/AST/BaseParser.hpp>
 #include <Ark/Compiler/AST/Node.hpp>
 #include <Ark/Compiler/AST/Import.hpp>
+#include <Ark/Compiler/Pass.hpp>
 #include <Ark/Utils.hpp>
 #include <Ark/Platform.hpp>
 
@@ -26,7 +27,7 @@
 
 namespace Ark::internal
 {
-    class ARK_API Parser : public BaseParser
+    class ARK_API Parser final : public BaseParser, public Pass
     {
     public:
         /**
@@ -37,7 +38,7 @@ namespace Ark::internal
 
         void process(const std::string& filename, const std::string& code);
 
-        [[nodiscard]] const Node& ast() const;
+        [[nodiscard]] const Node& ast() const noexcept override;
         [[nodiscard]] const std::vector<Import>& imports() const;
 
     private:
@@ -46,6 +47,9 @@ namespace Ark::internal
         std::vector<Import> m_imports;
         unsigned m_allow_macro_behavior;  ///< Toggled on when inside a macro definition, off afterward
 
+        // HACK so that the parser can be a pass and use the loggers
+        void process([[maybe_unused]] const Node&) override {}
+
         void run();
 
         Node& setNodePosAndFilename(Node& node, const std::optional<FilePosition>& cursor = std::nullopt) const;

include/Ark/Compiler/BytecodeReader.hpp

@@ -151,7 +151,7 @@ namespace Ark
         void display(BytecodeSegment segment = BytecodeSegment::All,
                      std::optional<uint16_t> sStart = std::nullopt,
                      std::optional<uint16_t> sEnd = std::nullopt,
-                     std::optional<uint16_t> cPage = std::nullopt);
+                     std::optional<uint16_t> cPage = std::nullopt) const;
 
         friend class Ark::State;
 

include/Ark/Compiler/Common.hpp

@@ -79,6 +79,18 @@ namespace Ark::internal
         "del"
     };
 
+    // This list is related to include/Ark/Compiler/Instructions.hpp
+    // The order is very important
+    constexpr std::array<std::string_view, 7> listInstructions = {
+        "list",
+        "append",
+        "concat",
+        "append!",
+        "concat!",
+        "pop",
+        "pop!"
+    };
+
     // This list is related to include/Ark/Compiler/Instructions.hpp
     // from FIRST_OPERATOR, to LAST_OPERATOR
     // The order is very important

include/Ark/Compiler/Compiler.hpp

@@ -2,7 +2,7 @@
  * @file Compiler.hpp
  * @author Alexandre Plateau ([email protected])
  * @brief ArkScript compiler is in charge of transforming the AST into bytecode
- * @version 2.1
+ * @version 3.0
  * @date 2020-10-27
  *
  * @copyright Copyright (c) 2020-2024
@@ -68,7 +68,6 @@ namespace Ark
 
         // tables: symbols, values, plugins and codes
         std::vector<internal::Node> m_symbols;
-        std::vector<std::string> m_defined_symbols;
         std::vector<std::string> m_plugins;
         std::vector<internal::ValTableElem> m_values;
         std::vector<std::vector<internal::Word>> m_code_pages;
@@ -119,43 +118,25 @@ namespace Ark
          * @brief Checking if a symbol is an operator
          *
          * @param name symbol name
-         * @return std::optional<std::size_t> position in the operators' list
+         * @return std::optional<uint8_t> operator instruction
          */
-        static std::optional<std::size_t> getOperator(const std::string& name) noexcept;
+        static std::optional<uint8_t> getOperator(const std::string& name) noexcept;
 
         /**
          * @brief Checking if a symbol is a builtin
          *
          * @param name symbol name
-         * @return std::optional<std::size_t> position in the builtins' list
+         * @return std::optional<uint16_t> builtin number
          */
-        static std::optional<std::size_t> getBuiltin(const std::string& name) noexcept;
+        static std::optional<uint16_t> getBuiltin(const std::string& name) noexcept;
 
         /**
-         * @brief Check if a symbol needs to be compiled to a specific instruction
+         * @brief Checking if a symbol is a list instruction
          *
          * @param name
-         * @return std::optional<internal::Instruction> corresponding instruction if it exists
+         * @return std::optional<internal::Instruction> list instruction
          */
-        static std::optional<internal::Instruction> getSpecific(const std::string& name) noexcept
-        {
-            if (name == "list")
-                return internal::Instruction::LIST;
-            if (name == "append")
-                return internal::Instruction::APPEND;
-            if (name == "concat")
-                return internal::Instruction::CONCAT;
-            if (name == "append!")
-                return internal::Instruction::APPEND_IN_PLACE;
-            if (name == "concat!")
-                return internal::Instruction::CONCAT_IN_PLACE;
-            if (name == "pop")
-                return internal::Instruction::POP_LIST;
-            if (name == "pop!")
-                return internal::Instruction::POP_LIST_IN_PLACE;
-
-            return std::nullopt;
-        }
+        static std::optional<internal::Instruction> getListInstruction(const std::string& name) noexcept;
 
         /**
          * Checks if a node is a list and has a keyboard as its first node, indicating if it's producing a value on the stack or not
@@ -174,14 +155,6 @@ namespace Ark
          */
         static bool isUnaryInst(internal::Instruction inst) noexcept;
 
-        /**
-         * @brief Compute specific instruction argument count
-         *
-         * @param inst
-         * @param previous
-         */
-        static uint16_t computeSpecificInstArgc(internal::Instruction inst, uint16_t previous) noexcept;
-
         /**
          * @brief Checking if a symbol may be coming from a plugin
          *
@@ -219,7 +192,7 @@ namespace Ark
         void compileExpression(const internal::Node& x, Page p, bool is_result_unused, bool is_terminal, const std::string& var_name = "");
 
         void compileSymbol(const internal::Node& x, Page p, bool is_result_unused);
-        void compileSpecific(const internal::Node& c0, const internal::Node& x, Page p, bool is_result_unused);
+        void compileListInstruction(const internal::Node& c0, const internal::Node& x, Page p, bool is_result_unused);
         void compileIf(const internal::Node& x, Page p, bool is_result_unused, bool is_terminal, const std::string& var_name);
         void compileFunction(const internal::Node& x, Page p, bool is_result_unused, const std::string& var_name);
         void compileLetMutSet(internal::Keyword n, const internal::Node& x, Page p);
@@ -254,27 +227,6 @@ namespace Ark
          * @return std::size_t
          */
         uint16_t addValue(std::size_t page_id, const internal::Node& current);
-
-        /**
-         * @brief Register a symbol as defined, so that later we can throw errors on undefined symbols
-         *
-         * @param sym
-         */
-        void addDefinedSymbol(const std::string& sym);
-
-        /**
-         * @brief Checks for undefined symbols, not present in the defined symbols table
-         *
-         */
-        void checkForUndefinedSymbol();
-
-        /**
-         * @brief Suggest a symbol of what the user may have meant to input
-         *
-         * @param str the string
-         * @return std::string
-         */
-        std::string offerSuggestion(const std::string& str) const;
     };
 }
 

include/Ark/Compiler/ImportSolver.hpp

@@ -1,31 +1,45 @@
+/**
+ * @file ImportSolver.hpp
+ * @author Alexandre Plateau ([email protected])
+ * @brief Handle imports, resolve them with modules and everything
+ * @version 2.0
+ * @date 2024-07-21
+ *
+ * @copyright Copyright (c) 2020-2024
+ *
+ */
+
 #ifndef ARK_COMPILER_IMPORTSOLVER_HPP
 #define ARK_COMPILER_IMPORTSOLVER_HPP
 
+#include <stack>
 #include <vector>
 #include <string>
 #include <filesystem>
 #include <unordered_map>
 
+#include <Ark/Compiler/Pass.hpp>
 #include <Ark/Compiler/AST/Node.hpp>
 #include <Ark/Compiler/AST/Import.hpp>
 #include <Ark/Compiler/AST/Module.hpp>
 
 namespace Ark::internal
 {
-    class ImportSolver final
+    class ImportSolver final : public Pass
     {
     public:
         ImportSolver(unsigned debug, const std::vector<std::filesystem::path>& libenv);
 
-        void process(const std::filesystem::path& root, const Node& origin_ast, const std::vector<Import>& origin_imports);
+        ImportSolver& setup(const std::filesystem::path& root, const std::vector<Import>& origin_imports);
+        void process(const Node& origin_ast) override;
 
-        [[nodiscard]] const Node& ast() const noexcept;
+        [[nodiscard]] const Node& ast() const noexcept override;
 
     private:
-        unsigned m_debug;
         std::vector<std::filesystem::path> m_libenv;
         std::filesystem::path m_root;  ///< Folder were the entry file is
         Node m_ast;
+        std::stack<Import> m_imports;
         std::unordered_map<std::string, Module> m_modules;  ///< Package to module map
         // TODO is this ok? is this fine? this is sort of ugly
         std::vector<std::string> m_imported;  ///< List of imports, in the order they were found and parsed

include/Ark/Compiler/Macros/Processor.hpp

@@ -14,6 +14,7 @@
 
 #include <Ark/Compiler/AST/Node.hpp>
 #include <Ark/Compiler/Macros/MacroScope.hpp>
+#include <Ark/Compiler/Pass.hpp>
 
 #include <unordered_map>
 #include <memory>
@@ -27,7 +28,7 @@ namespace Ark::internal
      * @brief The class handling the macros definitions and calls, given an AST
      *
      */
-    class MacroProcessor
+    class MacroProcessor final : public Pass
     {
     public:
         /**
@@ -42,19 +43,18 @@ namespace Ark::internal
          *
          * @param ast
          */
-        void process(const Node& ast);
+        void process(const Node& ast) override;
 
         /**
          * @brief Return the modified AST
          *
          * @return Node&
          */
-        [[nodiscard]] const Node& ast() const noexcept;
+        [[nodiscard]] const Node& ast() const noexcept override;
 
         friend class MacroExecutor;
 
     private:
-        unsigned m_debug;                  ///< The debug level
         Node m_ast;                        ///< The modified AST
         std::vector<MacroScope> m_macros;  ///< Handling macros in a scope fashion
         std::vector<std::shared_ptr<MacroExecutor>> m_executors;