[ci] use libra-cli bin for faster CI runs (#308)

0LNetworkCommunity · Aug 16, 2024 · 52b4602 · 52b4602
1 parent c6ec062
commit 52b4602
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 132 deletions.
diff --git a/.github/actions/build_env/action.yaml b/.github/actions/build_env/action.yaml
@@ -34,23 +34,12 @@ runs:
       shell: bash
       run: rustup component add rustfmt clippy
 
-    # - name: enable sccache
-    #   uses: 0o-de-lally/sccache-action@2dd73efb90f3c9113a0cab8d983df6eb2b21fd20
-
     - name: also rust cache
       uses: Swatinem/[email protected]
       with:
         shared-key: "libra"
         cache-on-failure: "true"
 
-    - name: export diem-node DIEM_FORGE_NODE_BIN_PATH
-      shell: bash
-      run: echo "DIEM_FORGE_NODE_BIN_PATH=${{github.workspace}}/diem-node" >> $GITHUB_ENV
-
-    - name: install diem-node (for smoke tests)
-      shell: bash
-      run: >
-        wget -O $DIEM_FORGE_NODE_BIN_PATH https://github.com/0LNetworkCommunity/diem/releases/latest/download/diem-node &&
-        sudo chmod 755 $DIEM_FORGE_NODE_BIN_PATH &&
-        echo $DIEM_FORGE_NODE_BIN_PATH &&
-        ls -l $DIEM_FORGE_NODE_BIN_PATH
+    # call own action for libra_cli
+    # TODO: figure out how to get the local action of this commit
+    - uses: 0LNetworkCommunity/libra-framework/.github/actions/get_cli@main
diff --git a/.github/actions/get_cli/action.yaml b/.github/actions/get_cli/action.yaml
@@ -0,0 +1,26 @@
+name: get cli bin
+description: use pre-build libra-cli bin for CI
+runs:
+  using: composite
+  steps:
+    - name: export diem-node DIEM_FORGE_NODE_BIN_PATH
+      shell: bash
+      run: |
+        echo "LIBRA_BIN=${{github.workspace}}/libra" >> $GITHUB_ENV
+        echo "DIEM_FORGE_NODE_BIN_PATH=${{github.workspace}}/libra" >> $GITHUB_ENV
+        # also have the file available in $PATH
+        echo ${{github.workspace}} >> $GITHUB_PATH
+
+    - name: install diem-node (for smoke tests)
+      shell: bash
+      run: >
+        wget -O $LIBRA_BIN https://github.com/0LNetworkCommunity/libra-framework/releases/download/ci-bins/libra &&
+        sudo chmod 755 $LIBRA_BIN
+    - name: print version
+      shell: bash
+      run: |
+        echo $LIBRA_BIN
+        echo $DIEM_FORGE_NODE_BIN_PATH
+        # for debugging, display the version, and ensure using the same aliases
+        ${{github.workspace}}/libra version
+        libra version
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -2,9 +2,8 @@ name: rust ci
 
 on:
   push:
-    tags:
+    tags: # only on releases, not RC, since we've tested already
       - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
     branches: ["**"] # glob pattern to allow slash /
   pull_request:
     types:
@@ -17,7 +16,6 @@ on:
     - cron: "30 00 * * *"
 
 env:
-  DIEM_FORGE_NODE_BIN_PATH: ${{github.workspace}}/diem-node
   LIBRA_CI: 1
   MODE_0L: "TESTNET"
 
@@ -28,17 +26,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: setup env
-        uses: ./.github/actions/build_env
-
-      - uses: Swatinem/[email protected]
-        with:
-          shared-key: "libra-framework"
-          cache-all-crates: true
+      - uses: ./.github/actions/get_cli
 
       - name: build framework
         working-directory: ./framework
-        run: cargo r release
+        # uses the libra binary to compile faster, without needing to compile the Move compiler
+        run: ${{github.workspace}}/libra move framework release
 
       - uses: actions/[email protected]
         with:
@@ -265,17 +258,17 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: setup env
-        if: ${{runner.environment == 'self-hosted'}}
+        # if: ${{runner.environment == 'self-hosted'}}
         uses: ./.github/actions/build_env
 
       - uses: actions/[email protected]
-        if: ${{runner.environment == 'self-hosted'}}
+        # if: ${{runner.environment == 'self-hosted'}}
         with:
           name: framework-build
           path: framework/
       - name: rescue
-        # if: always()
-        if: ${{runner.environment == 'self-hosted'}}
+        if: always()
+        # if: ${{runner.environment == 'self-hosted'}}
         working-directory: ./tools/rescue
         run: RUST_MIN_STACK=104857600 cargo test --no-fail-fast -- --test-threads=1
   storage:
@@ -294,7 +287,7 @@ jobs:
       - name: storage
         working-directory: ./tools/storage
         # TODO: not testing the V6 file parsing
-        run: cargo test --no-fail-fast e2e_
+        run: cargo test --no-fail-fast
 
   twin-testsuite:
     timeout-minutes: 60

diff --git a/.github/workflows/cleanliness.yaml b/.github/workflows/cleanliness.yaml
@@ -1,9 +1,8 @@
 name: cleanliness
 on:
   push:
-    tags:
+    tags: # only on releases, not RC, since we've tested already
       - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
     branches: ["**"] # glob pattern to allow slash /
   pull_request:
     types:
@@ -37,6 +36,10 @@ jobs:
           command: fmt
           args: --all -- --check
 
+      # TODO: clippy can share cache if build for tests is done prior
+      # - name: build for cache
+      #   run: cargo build --tests --workspace
+
       - name: clippy
         uses: actions-rs/cargo@v1
         with:

diff --git a/.github/workflows/formal.yaml b/.github/workflows/formal.yaml
@@ -1,9 +1,8 @@
 name: formal verification
 on:
   push:
-    tags:
+    tags: # only on releases, not RC, since we've tested already
       - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
     branches: ["**"] # glob pattern to allow slash /
   pull_request:
     types:
@@ -23,35 +22,23 @@ jobs:
   formal:
     runs-on: ubuntu-latest
     steps:
-      #   # NOTE: for debugging CI this allow shell access to github runner. Will print out tmate.io terminal url
+      # NOTE: for debugging CI this allow shell access to github runner. Will print out tmate.io terminal url
       # - name: Setup tmate session
       #   uses: mxschmitt/action-tmate@v3
       #   with:
       #     detached: true
-      #   timeout-minutes: 15
+      #   timeout-minutes: 3
 
       - uses: actions/checkout@v3
 
       - name: setup env
         uses: ./.github/actions/build_env
 
-      - uses: Swatinem/[email protected]
-        with:
-          shared-key: "libra-framework"
-          cache-all-crates: true
-
       - name: install prover dependencies
         run: |
           bash util/dev_setup.sh -byp
 
-      - name: install diem (for move tests)
-        run: |
-          wget -O ${{github.workspace}}/diem https://github.com/0LNetworkCommunity/diem/releases/latest/download/diem &&
-          chmod +x ${{github.workspace}}/diem &&
-          cp ${{github.workspace}}/diem ~/.cargo/bin
-
-      # Move framework tests
-      # TODO:
+      # Move prover tests
       - name: prover tests
         working-directory: ./framework
-        run: make prove
+        run: make -f prover.mk prove
diff --git a/.github/workflows/move.yaml b/.github/workflows/move.yaml
@@ -2,9 +2,8 @@ name: move framework tests
 
 on:
   push:
-    tags:
+    tags: # only on releases, not RC, since we've tested already
       - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
     branches: ["**"] # glob pattern to allow slash /
   pull_request:
     types:
@@ -27,17 +26,10 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - name: install diem (for move tests)
-        run: >
-          wget -O ${{github.workspace}}/diem https://github.com/0LNetworkCommunity/diem/releases/latest/download/diem &&
-          chmod +x ${{github.workspace}}/diem
-
-      - uses: Swatinem/[email protected]
-        with:
-            shared-key: "libra-framework"
-            cache-all-crates: true
+      - name: fetch libra cli
+        uses: ./.github/actions/get_cli
 
       # Move framework tests
       - name: move framework
         working-directory: ./framework/libra-framework
-        run: ${{github.workspace}}/diem move test
+        run: $LIBRA_BIN move test
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -1,16 +1,14 @@
-name: publish bin
+name: publish cli
 on:
   push:
-    tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
-    branches:
-      - "ci"
+    tags: # run this also on release candidates
+      - "[0-9]+.[0-9]+.[0-9]*"
+      # make binaries which may be ahead of releases to use in CI jobs
+      - "ci-bins"
 jobs:
   publish:
     permissions:
       write-all
-      # contents: write
     name: publish
     runs-on: ubuntu-latest
     steps:
@@ -20,41 +18,14 @@ jobs:
       #   with:
       #     detached: true
       #   timeout-minutes: 15
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
-
-      - uses: dtolnay/[email protected]
-        with:
-          components: rustfmt
-
-      ######## CACHE ########
-      - name: system packages
-        uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: build-essential ca-certificates clang curl git libpq-dev libssl-dev pkg-config lsof lld libgmp-dev
-          version: 1.0
 
       - name: checkout
         uses: actions/checkout@v3
 
-      - name: sccache
-        uses: 0o-de-lally/sccache-action@local
+      - name: setup env
+        uses: ./.github/actions/build_env
 
-      # note: building in the same cargo command will lead to "feature unification", which leads to a `diem-node` binary which fails.
-      - name: libra release build
+      - name: build libra cli release
         # size and performance optimized binary with profile.cli
         run: cargo b --release -p libra
 
@@ -66,17 +37,3 @@ jobs:
           tag: ${{ github.ref }}
           overwrite: true
           file_glob: true
-
-      # TODO
-      # - name: libra-framework release build
-      # # size and performance optimized binary with profile.cli
-      #   run: cargo b --release -p libra-framework
-
-      # - name: CLI publish
-      #   uses: svenstaro/upload-release-action@v2
-      #   with:
-      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
-      #     file: target/release/libra-framework
-      #     tag: ${{ github.ref }}
-      #     overwrite: true
-      #     file_glob: true