Merge pull request #443 from ArkScript-lang/feat/code-formatter

Feat/code formatter

.github/workflows/ci.yml

@@ -89,9 +89,9 @@ jobs:
             artifact: "windows-msvc-22"
           }
           - {
-            os: macos-latest, name: "MacOS Clang 14",
-            artifact: "macos-clang-14",
-            compiler: clang, compiler_version: 14, sanitizers: "On"
+            os: macos-latest, name: "MacOS Clang 15",
+            artifact: "macos-clang-15",
+            compiler: clang, compiler_version: 15, sanitizers: "On"
           }
 
     steps:
@@ -156,7 +156,7 @@ jobs:
           - { os: ubuntu-latest,  name: "Ubuntu Clang 15", artifact: "ubuntu-clang-15" }
           - { os: ubuntu-latest,  name: "Ubuntu GCC 13",   artifact: "ubuntu-gcc-13" }
           - { os: windows-latest, name: "Windows VS 2022", artifact: "windows-msvc-22", }
-          - { os: macos-latest,   name: "MacOS Clang 14",  artifact: "macos-clang-14", }
+          - { os: macos-latest,   name: "MacOS Clang 15",  artifact: "macos-clang-15", }
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/setup-compilers/action.yaml

@@ -26,8 +26,8 @@ runs:
       shell: bash
       run: |
         if [[ '${{ startsWith(inputs.os_name, 'macos') }}' == 'true' ]]; then
-          echo "cc=${{ inputs.compiler }}" >> $GITHUB_OUTPUT
-          echo "cxx=${{ inputs.compiler }}++" >> $GITHUB_OUTPUT
+          echo "cc=/usr/local/opt/llvm@${{ inputs.compiler_version }}/bin/${{ inputs.compiler }}" >> $GITHUB_OUTPUT
+          echo "cxx=/usr/local/opt/llvm@${{ inputs.compiler_version }}/bin/${{ inputs.compiler }}++" >> $GITHUB_OUTPUT
         elif [[ '${{ inputs.compiler }}' == 'clang' ]]; then
           echo "cc=clang-${{ inputs.compiler_version }}" >> $GITHUB_OUTPUT
           echo "cxx=clang++-${{ inputs.compiler_version }}" >> $GITHUB_OUTPUT
@@ -51,6 +51,12 @@ runs:
           libc++-${{ inputs.compiler_version }}-dev libc++abi-${{ inputs.compiler_version }}-dev  \
           clang-tools-${{ inputs.compiler_version }}
 
+    - name: Update LLVM compilers
+      if: startsWith(inputs.os_name, 'macos')
+      shell: bash
+      run: |
+        brew install llvm@${{ inputs.compiler_version }}
+
     - name: Setup Windows environment
       uses: ilammy/msvc-dev-cmd@v1
       if: startsWith(inputs.os_name, 'windows')

CHANGELOG.md

@@ -8,6 +8,8 @@
 - added the padding/instruction/argumentation values when displaying instructions in the bytecode reader
 - `$repr` macro to get a string representation of a given node
 - added boost-ext/ut to write unit tests in C++
+- basic ArkScript code formatter, available through the CLI: `arkscript -f|--format`
+- comments are now tracked in the AST and attached to the nearest node below them
 
 ### Changed
 - instructions are on 4 bytes: 1 byte for the instruction, 1 byte of padding, 2 bytes for an immediate argument

CMakeLists.txt

@@ -109,10 +109,6 @@ target_include_directories(ArkReactor
 target_link_libraries(ArkReactor PUBLIC termcolor)
 
 if (UNIX OR LINUX)
-    if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG)
-        target_link_libraries(ArkReactor PUBLIC stdc++fs)
-    endif()
-
     find_package(Threads)
     target_link_libraries(ArkReactor PRIVATE ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
 endif()
@@ -180,10 +176,15 @@ endif()
 
 # TODO: consider using ctest
 if (ARK_TESTS)
-    file(GLOB_RECURSE UT_SOURCES ${ark_SOURCE_DIR}/tests/unittests/*.cpp)
+    file(GLOB_RECURSE UT_SOURCES
+            ${ark_SOURCE_DIR}/tests/unittests/*.cpp
+            ${ark_SOURCE_DIR}/lib/fmt/src/format.cc
+            ${ark_SOURCE_DIR}/src/arkscript/Formatter.cpp
+            ${ark_SOURCE_DIR}/src/arkscript/JsonCompiler.cpp)
     add_executable(unittests ${UT_SOURCES})
 
     add_subdirectory(${ark_SOURCE_DIR}/lib/ut)
+    target_include_directories(unittests PUBLIC ${ark_SOURCE_DIR}/include)
     target_link_libraries(unittests PUBLIC ArkReactor termcolor ut)
 
     add_compile_definitions(BOOST_UT_DISABLE_MODULE)
@@ -194,11 +195,8 @@ endif()
 if (ARK_BUILD_EXE)
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES
-        ${ark_SOURCE_DIR}/src/arkscript/REPL/Utils.cpp
-        ${ark_SOURCE_DIR}/src/arkscript/REPL/Repl.cpp
-        ${ark_SOURCE_DIR}/lib/fmt/src/format.cc
-        ${ark_SOURCE_DIR}/src/arkscript/main.cpp)
-
+        ${ark_SOURCE_DIR}/src/arkscript/*.cpp
+        ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
     add_executable(arkscript ${EXE_SOURCES})
 
     if (MSVC)
@@ -213,9 +211,7 @@ if (ARK_BUILD_EXE)
     add_subdirectory("${ark_SOURCE_DIR}/lib/replxx" EXCLUDE_FROM_ALL)
     add_subdirectory("${ark_SOURCE_DIR}/lib/clipp" EXCLUDE_FROM_ALL)
 
-    target_include_directories(arkscript PUBLIC "${ark_SOURCE_DIR}/src/arkscript/")
     target_link_libraries(arkscript PUBLIC ArkReactor replxx clipp termcolor)
-
     target_compile_features(arkscript PRIVATE cxx_std_20)
 
     enable_lto(arkscript)

README.md

@@ -186,20 +186,35 @@ SYNOPSIS
         arkscript --dev-info 
         arkscript -e <expression> 
         arkscript -c <file> [-d] 
+        arkscript <file> [-d] [-L <lib_dir>]  
+        arkscript -f <file> [--dry-run] 
+        arkscript --ast <file> [-d] [-L <lib_dir>] 
         arkscript -bcr <file> -on 
         arkscript -bcr <file> -a [-s <start> <end>] 
         arkscript -bcr <file> -st [-s <start> <end>] 
         arkscript -bcr <file> -vt [-s <start> <end>] 
         arkscript -bcr <file> [-cs] [-p <page>] [-s <start> <end>] 
-        arkscript <file> [-d] [-L <lib_dir>]  
 
 OPTIONS
         -h, --help                  Display this message
         -v, --version               Display ArkScript version and exit
         --dev-info                  Display development information and exit
         -e, --eval                  Evaluate ArkScript expression
+
         -c, --compile               Compile the given program to bytecode, but do not run
         -d, --debug...              Increase debug level (default: 0)
+
+        -L, --lib                   Set the location of the ArkScript standard library. Paths can be
+                                    delimited by ';'
+
+        -f, --format                Format the given source file in place
+        --dry-run                   Do not modify the file, only print out the changes
+
+        --ast                       Compile the given program and output its AST as JSON to stdout
+        -d, --debug...              Increase debug level (default: 0)
+        -L, --lib                   Set the location of the ArkScript standard library. Paths can be
+                                    delimited by ';'
+
         -bcr, --bytecode-reader     Launch the bytecode reader
         -on, --only-names           Display only the bytecode segments names and sizes
         -a, --all                   Display all the bytecode segments (default)
@@ -208,8 +223,9 @@ OPTIONS
         -cs, --code                 Display only the code segments
         -p, --page                  Set the bytecode reader code segment to display
         -s, --slice                 Select a slice of instructions in the bytecode
-        -L, --lib                   Set the location of the ArkScript standard library. Paths can be
-                                    delimited by ';'
+
+VERSION
+        4.0.0-86587c14
 
 LICENSE
         Mozilla Public License 2.0

include/Ark/Compiler/AST/BaseParser.hpp

@@ -5,6 +5,7 @@
 #include <exception>
 #include <stdexcept>
 #include <utility>
+#include <vector>
 #include <initializer_list>
 
 #include <Ark/Platform.hpp>
@@ -15,8 +16,8 @@ namespace Ark::internal
 {
     struct FilePosition
     {
-        std::size_t row;
-        std::size_t col;
+        std::size_t row = 0;
+        std::size_t col = 0;
     };
 
     class ARK_API BaseParser
@@ -25,17 +26,22 @@ namespace Ark::internal
         BaseParser() = default;
 
     private:
-        std::string m_filename;
         std::string m_str;
+        std::vector<std::pair<std::string::iterator, std::size_t>> m_it_to_row;
         std::string::iterator m_it, m_next_it;
         utf8_char_t m_sym;
+        FilePosition m_filepos;
+
+        void registerNewLine(std::string::iterator it, std::size_t row);
 
         /*
             getting next character and changing the values of count/row/col/sym
         */
         void next();
 
     protected:
+        std::string m_filename;
+
         void initParser(const std::string& filename, const std::string& code);
 
         FilePosition getCursor();
@@ -69,8 +75,9 @@ namespace Ark::internal
         bool space(std::string* s = nullptr);
         bool inlineSpace(std::string* s = nullptr);
         bool endOfLine(std::string* s = nullptr);
-        bool comment();
-        bool newlineOrComment();
+        bool comment(std::string* s = nullptr);
+        bool spaceComment(std::string* s = nullptr);
+        bool newlineOrComment(std::string* s = nullptr);
         bool prefix(char c);
         bool suffix(char c);
         bool number(std::string* s = nullptr);

include/Ark/Compiler/AST/Node.hpp

@@ -127,6 +127,15 @@ namespace Ark::internal
          */
         void setFilename(const std::string& filename) noexcept;
 
+        /**
+         * @brief Set the comment field with the nearest comment before this node
+         * @param comment
+         * @return Node& reference to this node after updating it
+         */
+        Node& attachNearestCommentBefore(const std::string& comment);
+
+        Node& attachCommentAfter(const std::string& comment);
+
         /**
          * @brief Get the line at which this node was created
          *
@@ -148,6 +157,14 @@ namespace Ark::internal
          */
         [[nodiscard]] const std::string& filename() const noexcept;
 
+        /**
+         * @brief Return the comment attached to this node, if any
+         * @return const std::string&
+         */
+        [[nodiscard]] const std::string& comment() const noexcept;
+
+        [[nodiscard]] const std::string& commentAfter() const noexcept;
+
         /**
          * @brief Compute a representation of the node without any comments or additional sugar, colors, types
          * @return String representation of the node
@@ -165,6 +182,8 @@ namespace Ark::internal
         // position of the node in the original code, useful when it comes to parser errors
         std::size_t m_line = 0, m_col = 0;
         std::string m_filename;
+        std::string m_comment;
+        std::string m_after_comment;  ///< Comment after node
     };
 
     ARK_API std::ostream& operator<<(std::ostream& os, const std::vector<Node>& node) noexcept;