Merge pull request #425 from ArkScript-lang/feat/parser

Integrating the new parser

.github/workflows/ci.yml

@@ -139,38 +139,11 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Update GNU compilers
-      if: startsWith(matrix.config.name, 'Ubuntu GCC')
-      shell: bash
-      run: |
-        sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test
-        sudo apt-get -yq install ${{ matrix.config.cc }} ${{ matrix.config.cxx }}
-
-    - name: Update LLVM compilers
-      if: startsWith(matrix.config.name, 'Ubuntu Clang')
-      shell: bash
-      run: |
-        version=`echo ${{ matrix.config.cc }} | cut -c 7-`
-        sudo apt-get install -y clang-${version} lld-${version} libc++-${version}-dev libc++abi-${version}-dev clang-tools-${version}
+    - name: Setup compilers
+      uses: ./.github/workflows/setup-compilers
 
-    - name: Install MacOS dependencies
-      if: startsWith(matrix.config.name, 'MacOS')
-      shell: bash
-      run: env HOMEBREW_NO_AUTO_UPDATE=1 brew install openssl
-
-    - uses: ilammy/msvc-dev-cmd@v1
-      if: startsWith(matrix.config.name, 'Windows')
-
-    - name: Download Windows dependencies
-      if: startsWith(matrix.config.name, 'Windows')
-      shell: pwsh
-      run: |
-        Invoke-RestMethod -Uri https://www.sqlite.org/2022/sqlite-dll-win64-x64-${Env:SQLITE_VERSION}.zip -OutFile sqlite.zip
-        Invoke-RestMethod -Uri https://www.sqlite.org/2022/sqlite-amalgamation-${Env:SQLITE_VERSION}.zip  -OutFile amalgation.zip
-        Expand-Archive sqlite.zip -DestinationPath sqlite_lib
-        Expand-Archive amalgation.zip -DestinationPath sqlite_code
-        cd sqlite_lib
-        lib /DEF:sqlite3.def /OUT:sqlite3.lib /MACHINE:x64
+    - name: Setup dependencies
+      uses: ./.github/workflows/setup-deps
 
     - name: Configure CMake Ark
       shell: bash
@@ -181,7 +154,7 @@ jobs:
           -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
           -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }} \
-          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On
+          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On -DARK_BUILD_PARSER_TESTS=On
 
     - name: Add SQLite deps
       if: startsWith(matrix.config.name, 'Windows')
@@ -195,7 +168,7 @@ jobs:
       shell: bash
       run: cmake --build build --config $BUILD_TYPE
 
-    - name: Configure CMake Integration tests
+    - name: Configure & build CMake Integration tests
       shell: bash
       run: |
         cd tests/cpp
@@ -204,38 +177,31 @@ jobs:
           -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
           -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }}
-
-    - name: Build Integration tests
-      shell: bash
-      run: cd tests/cpp && cmake --build build --config $BUILD_TYPE
-
-    - name: Organize files for upload
-      if: startsWith(matrix.config.name, 'Ubuntu') || startsWith(matrix.config.name, 'MacOS')
-      shell: bash
-      run: |
-        mkdir -p artifact/lib/std
-        cp build/arkscript artifact
-        cp build/libArkReactor.* artifact
-        cp lib/*.arkm artifact/lib
-        cp lib/std/*.ark artifact/lib/std
-        rm -rf artifact/lib/std/{.git,.github,tests/__arkscript__}
+        cmake --build build --config $BUILD_TYPE
 
     - name: Organize files for upload
-      if: startsWith(matrix.config.name, 'Windows')
       shell: bash
       run: |
         mkdir -p artifact/lib/std
-        cp build/$BUILD_TYPE/arkscript.exe artifact
-        cp build/$BUILD_TYPE/ArkReactor.dll artifact
+        # Linux/MacOS
+        cp build/arkscript artifact || true
+        cp build/parser artifact || true
+        cp build/libArkReactor.* artifact || true
+        # Windows
+        cp build/$BUILD_TYPE/arkscript.exe artifact || true
+        cp build/$BUILD_TYPE/parser.exe artifact || true
+        cp build/$BUILD_TYPE/ArkReactor.dll artifact  || true
+        # Generic
         cp lib/*.arkm artifact/lib
         cp lib/std/*.ark artifact/lib/std
         rm -rf artifact/lib/std/{.git,.github,tests/__arkscript__}
 
     - name: Organize temp artifact
       shell: bash
       run: |
-        mkdir -p temp/
+        mkdir -p temp/parser/
         cp -r tests/cpp temp/
+        cp -r tests/parser temp/
 
     - name: Upload artifact
       uses: actions/[email protected]
@@ -252,7 +218,7 @@ jobs:
 
   tests:
     runs-on: ${{ matrix.config.os }}
-    name: Test on ${{ matrix.config.name }}
+    name: Tests on ${{ matrix.config.name }}
     needs: [build]
 
     strategy:
@@ -279,46 +245,44 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Download artifact
-      id: download
-      uses: actions/[email protected]
-      with:
-        name: ${{ matrix.config.artifact }}
-        path: build
+    - name: Setup tests
+      uses: ./.github/workflows/setup-tests
 
-    - name: Download temp artifact
-      id: download-artifact
-      uses: actions/[email protected]
-      with:
-        name: temp-${{ matrix.config.artifact }}
-        path: artifact
+    - name: Parser tests
+      shell: bash
+      run: |
+        export ASAN_OPTIONS=detect_odr_violation=0
+        (cd tests/parser/tests ; bash ./run)
 
-    - name: Update GNU compilers
-      if: startsWith(matrix.config.name, 'Ubuntu GCC')
+    - name: Integration tests
       shell: bash
       run: |
-        sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test
-        sudo apt-get -yq install libstdc++6
+        export ASAN_OPTIONS=detect_odr_violation=0
+        (cd tests/cpp ; bash ./run-tests)
 
-    - shell: bash
+    - name: AST tests
+      shell: bash
       run: |
-        mv artifact/cpp/out tests/cpp/
-        mv build/lib/*.arkm lib/
-        chmod u+x build/arkscript tests/cpp/out/*
+        export ASAN_OPTIONS=use_odr_indicator=1
+        (cd tests/ast ; bash ./run-tests)
 
-    - name: Pre-test
-      if: startsWith(matrix.config.name, 'Windows')
+    - name: Unit tests
       shell: bash
       run: |
-        mkdir -p tests/cpp/out
-        cp build/*.dll tests/cpp/out/
+        export ASAN_OPTIONS=detect_odr_violation=0
+        (cd tests/arkscript ; bash ./run-tests)
 
-    - name: Tests
-      if: steps.download.outcome == 'success' && steps.download-artifact.outcome == 'success'
+    #- name: Modules tests
+    #  shell: bash
+    #  run: |
+    #    export ASAN_OPTIONS=detect_odr_violation=0
+    #    (source ./lib/modules/.github/run-tests)
+
+    - name: Runtime error message generation tests
       shell: bash
       run: |
-        export ASAN_OPTIONS=use_odr_indicator=1
-        bash .github/launch-tests
+        export ASAN_OPTIONS=detect_odr_violation=0
+        (cd tests/errors ; bash ./run-tests)
 
   valgrind:
     runs-on: ubuntu-latest
@@ -337,14 +301,11 @@ jobs:
         name: "ubuntu-clang-11-valgrind"
         path: build
 
-    - shell: bash
-      run: |
-        mv build/lib/*.arkm lib/
-        chmod u+x build/arkscript
-
     - name: Update LLVM compilers
       shell: bash
       run: |
+        mv build/lib/*.arkm lib/
+        chmod u+x build/arkscript
         sudo apt-get update --fix-missing
         sudo apt-get install -y clang-11 lld-11 libc++-11-dev libc++abi-11-dev clang-tools-11 valgrind
 

.github/workflows/setup-compilers/action.yaml

@@ -0,0 +1,22 @@
+---
+name: "Update compilers"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Update GNU compilers
+      if: startsWith(matrix.config.name, 'Ubuntu GCC')
+      shell: bash
+      run: |
+        sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test
+        sudo apt-get -yq install ${{ matrix.config.cc }} ${{ matrix.config.cxx }}
+
+    - name: Update LLVM compilers
+      if: startsWith(matrix.config.name, 'Ubuntu Clang')
+      shell: bash
+      run: |
+        version=`echo ${{ matrix.config.cc }} | cut -c 7-`
+        sudo apt-get install -y clang-${version} lld-${version} libc++-${version}-dev libc++abi-${version}-dev clang-tools-${version}
+
+    - uses: ilammy/msvc-dev-cmd@v1
+      if: startsWith(matrix.config.name, 'Windows')

.github/workflows/setup-deps/action.yaml

@@ -0,0 +1,21 @@
+---
+name: "Install compilers and dependencies"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install MacOS dependencies
+      if: startsWith(matrix.config.name, 'MacOS')
+      shell: bash
+      run: env HOMEBREW_NO_AUTO_UPDATE=1 brew install openssl
+
+    - name: Download Windows dependencies
+      if: startsWith(matrix.config.name, 'Windows')
+      shell: pwsh
+      run: |
+        Invoke-RestMethod -Uri https://www.sqlite.org/2022/sqlite-dll-win64-x64-${Env:SQLITE_VERSION}.zip -OutFile sqlite.zip
+        Invoke-RestMethod -Uri https://www.sqlite.org/2022/sqlite-amalgamation-${Env:SQLITE_VERSION}.zip  -OutFile amalgation.zip
+        Expand-Archive sqlite.zip -DestinationPath sqlite_lib
+        Expand-Archive amalgation.zip -DestinationPath sqlite_code
+        cd sqlite_lib
+        lib /DEF:sqlite3.def /OUT:sqlite3.lib /MACHINE:x64

.github/workflows/setup-tests/action.yaml

@@ -0,0 +1,42 @@
+---
+name: "Setup tests"
+description: "Unpack necessary artifacts, updates compilers"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      id: download
+      uses: actions/[email protected]
+      with:
+        name: ${{ matrix.config.artifact }}
+        path: build
+
+    - name: Download temp artifact
+      id: download-artifact
+      uses: actions/[email protected]
+      with:
+        name: temp-${{ matrix.config.artifact }}
+        path: artifact
+
+    - name: Update GNU compilers
+      if: startsWith(matrix.config.name, 'Ubuntu GCC')
+      shell: bash
+      run: |
+        sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test
+        sudo apt-get -yq install libstdc++6
+
+    - shell: bash
+      run: |
+        mv artifact/cpp/out tests/cpp/
+        mv build/lib/*.arkm lib/
+        chmod u+x build/arkscript tests/cpp/out/*
+        cp -r artifact/parser/* tests/parser/
+        cp -r build tests/parser/ && ls tests/parser/build/
+        chmod u+x tests/parser/build/parser
+
+    - shell: bash
+      if: startsWith(matrix.config.name, 'Windows')
+      run: |
+        cp build/*.dll tests/cpp/out/
+        cp build/*.dll tests/parser/build/

.gitignore

@@ -4,7 +4,6 @@
 
 # Personal utilities
 warnings.log
-cformat.ps1
 
 # ArkScript
 include/Ark/Constants.hpp
@@ -28,6 +27,7 @@ afl/
 .cache/
 build/
 ninja/
+cmake-build-*/
 
 # Prerequisites
 *.d
@@ -47,10 +47,6 @@ ninja/
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
-
 # Compiled Static libraries
 *.lai
 *.la
@@ -64,3 +60,6 @@ ninja/
 
 # MacOS files
 .DS_store
+
+# Visual Studio
+CmakeSettings.json

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "lib/utf8_decoder"]
-	path = lib/utf8_decoder
-	url = https://github.com/PierrePharel/utf8_decoder.git
 [submodule "lib/std"]
 	path = lib/std
 	url = https://github.com/ArkScript-lang/std.git

CHANGELOG.md

@@ -5,6 +5,7 @@
 - more tests for the io builtins
 - added lines and code coloration in the error context
 - new dependency: fmtlib
+- added the padding/instruction/argumentation values when displaying instructions in the bytecode reader
 
 ### Changed
 - instructions are on 4 bytes: 1 byte for the instruction, 1 byte of padding, 2 bytes for an immediate argument
@@ -43,6 +44,11 @@
 - fixed a bug in the macro processor where macros were deleted when they shouldn't
 - fixed a bug where macro functions with no argument would crash the macro processor
 
+### Removed
+
+### Deprecated
+
+
 ## [3.4.0] - 2022-09-12
 ### Added
 - added new `async` and `await` builtins