CI musl linux + Zig CC (aka zig linker) (#580)

* ci linux-musl * don't use zigbuild on macos * update deps * v6.1.33
gosh-sh · Sep 19, 2023 · c160ad4 · c160ad4
1 parent ecd5ee6
commit c160ad4
Show file tree

Hide file tree

Showing 5 changed files with 149 additions and 112 deletions.
diff --git a/.github/workflows/prerelease.yaml b/.github/workflows/prerelease.yaml
@@ -1,41 +1,35 @@
 name: Prerelease
 
+# IMPORTANT: This workflow utilizes caching for previous build artifacts. In the
+# event of a full/partial rebuild, remember to manually remove corresponding
+# cache records from the GitHub Actions admin panel.
+#
+# Benefit of this caching strategy is that it allows for individual cache
+# invalidation for each binary
+
 on:
   workflow_dispatch:
   push:
     branches:
-      - dev
+      - main
+      - 'ci/**'
+    tags:
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
 
 permissions:
   contents: write
 
 env:
+  release: ${{ startsWith(github.ref, 'refs/tags/') || github.repository == 'gosh-sh/gosh' }}
   CARGO_TERM_COLOR: always
-  CARGO_INCREMENTAL: 0  # important for cache size too
-  CARGO_NET_RETRY: 10
-  RUST_BACKTRACE: full
-  RUSTUP_MAX_RETRIES: 10
-  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
-  CARGO_PROFILE_RELEASE_LTO: true
-  CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: x86_64-linux-gnu-gcc
-  CC_x86_64_unknown_linux_gnu: x86_64-linux-gnu-gcc
-  CXX_x86_64_unknown_linux_gnu: x86_64-linux-gnu-g++
-  CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
-  CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
-  CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
-  RUSTFLAGS: -C target-feature=+crt-static
 
 jobs:
   build:
-    runs-on: "${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        name:
-          - git-remote-gosh-darwin-arm64
-          - git-remote-gosh-darwin-amd64
-          - git-remote-gosh-linux-arm64
-          - git-remote-gosh-linux-amd64
         bin:
           - git-remote-gosh
           - git-remote-gosh_v1_0_0
@@ -46,51 +40,60 @@ jobs:
           - git-remote-gosh_v5_1_0
           - git-remote-gosh_v6_0_0
           - git-remote-gosh_v6_1_0
+        result:
+          - git-remote-gosh-darwin-arm64
+          - git-remote-gosh-darwin-amd64
+          - git-remote-gosh-linux-arm64
+          - git-remote-gosh-linux-amd64
+
         include:
-          ## bin
+          # targets
+          - result: git-remote-gosh-darwin-arm64
+            target: aarch64-apple-darwin
+            os: macos-12
+
+          - result: git-remote-gosh-darwin-amd64
+            target: x86_64-apple-darwin
+            os: macos-12
+
+          - result: git-remote-gosh-linux-arm64
+            target: aarch64-unknown-linux-musl
+            os: ubuntu-22.04
+
+          - result: git-remote-gosh-linux-amd64
+            target: x86_64-unknown-linux-musl
+            os: ubuntu-22.04
+
+          # cargo projects
           - bin: git-remote-gosh
             dir: gosh-dispatcher
           - bin: git-remote-gosh_v1_0_0
             dir: v1_x/git-remote-gosh
-            copy_abi: true
           - bin: git-remote-gosh_v2_0_0
             dir: v2_x/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v3_0_0
             dir: v3_x/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v4_0_0
             dir: v4_x/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v5_0_0
             dir: v5_x/v5.0.0/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v5_1_0
             dir: v5_x/v5.1.0/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v6_0_0
             dir: v6_x/v6.0.0/git-remote-gosh
-            copy_abi: true
+            cache: true
           - bin: git-remote-gosh_v6_1_0
             dir: v6_x/v6.1.0/git-remote-gosh
-            copy_abi: true
 
-          ## name
-          - name: git-remote-gosh-darwin-arm64
-            os: macos-12
-            target: aarch64-apple-darwin
-          - name: git-remote-gosh-darwin-amd64
-            os: macos-12
-            target: x86_64-apple-darwin
-          - name: git-remote-gosh-linux-arm64
-            os: ubuntu-22.04
-            target: aarch64-unknown-linux-gnu
-          - name: git-remote-gosh-linux-amd64
-            os: ubuntu-22.04
-            target: x86_64-unknown-linux-gnu
-          # - name: git-remote-gosh-windows-amd64
-          #   os: windows-2019
-          #   target: x86_64-pc-windows-msvc
+    env:
+      CACHE: false  # default env value
+      CACHE_RESULT_DIR: ${{ github.workspace }}/.result/${{ matrix.result }}
+      CACHE_RESULT_BIN: ${{ github.workspace }}/.result/${{ matrix.result }}/${{ matrix.bin }}
 
     steps:
       - name: Checkout repo
@@ -101,68 +104,98 @@ jobs:
         run: |
           echo "VERSION=$(cat version.json | jq -r .version)" >> "$GITHUB_OUTPUT"
 
-      # Cache
-      - name: Set up cargo cache
+      - name: Setup result cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CACHE_RESULT_DIR }}
+          key: result-${{ matrix.target }}-${{ matrix.bin }}
+
+      - name: Check if can use cache
+        if: matrix.cache
+        run: |
+          if [[ -f ${{ env.CACHE_RESULT_BIN }} ]] ; then
+            echo "CACHE=true" >> $GITHUB_ENV
+          fi
+
+      - name: Setup cargo cache
+        if: env.CACHE != 'true'
         uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo
+          key: builder-${{ runner.os }}-cargo
 
-      - uses: actions-rs/toolchain@v1
+      - name: Install Rust
+        if: env.CACHE != 'true'
+        uses: dtolnay/rust-toolchain@stable
         with:
-          profile: minimal
-          toolchain: stable
-          target: ${{ matrix.target }}
-          override: true
+          targets: ${{ matrix.target }}
 
-      - name: Install Linux packages
-        if: startsWith('ubuntu-', matrix.os)
+      - name: Install Linux tools
+        if: env.CACHE != 'true' && startsWith(matrix.os, 'ubuntu-')
         env:
-          packages: >
+          packages: >-
+            build-essential
+            cmake
+            curl
+            clang
             librocksdb-dev
-            libssl-dev
+            protobuf-compiler
+            python3-pip
         run: |
-          sudo apt-get update -y && sudo apt-get install -y $packages
-
-      - name: Add cross compile Linux tools
-        if: ${{ matrix.target == 'aarch64-unknown-linux-gnu' }}
+          sudo apt update -yq
+          sudo apt install -yq $packages
+          pip3 install ziglang
+
+      ## TODO: return back when openssl is removed from old repos (v1 v2 v3)
+      # - name: Install MacOS tools
+      #   if: env.CACHE != 'true' && startsWith(matrix.os, 'macos-')
+      #   run: |
+      #     brew install zig protobuf openssl llvm
+
+      - name: Install common tools
+        if: env.CACHE != 'true'
         run: |
-          sudo apt-get update -y && sudo apt-get install -y g++-aarch64-linux-gnu libc6-dev-arm64-cross gcc-aarch64-linux-gnu
-          rustup target add aarch64-unknown-linux-gnu
-
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        ## NOTE: workaround rate-limit
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Copy ABI
-        if: ${{ matrix.copy_abi }}
-        working-directory: ${{ matrix.dir }}
-        run: make copy_abi
+          [[ -f ~/.cargo/bin/cargo-zigbuild ]] || cargo install cargo-zigbuild
 
       - name: Build Binary ${{ matrix.dir }}/target/${{ matrix.target }}/release/${{ matrix.bin }}
         working-directory: ${{ matrix.dir }}
-        if: ${{ true }}
+        if: env.CACHE != 'true'
+        env:
+          CARGO_INCREMENTAL: 0  # important for cache size too
+          CARGO_NET_RETRY: 10
+          RUST_BACKTRACE: full
+          RUSTUP_MAX_RETRIES: 10
+
+        # IMPORTANT: don't use zigbuild on MacOS until openssl dependency present
+        # Right now they are: v1, v2, v3
+        # It doesn't cross compile (actually doesn't link) to aarch64 from x86_64
+        # rustls TLS works just fine
         run: |
-          cargo build --release --target=${{ matrix.target }}
+          if [[ "${{ runner.os }}" == "macOS" ]]; then
+            cargo build -r --target ${{ matrix.target }}
+          else
+            cargo zigbuild -r --target ${{ matrix.target }}
+          fi
+
           ls -lA target/${{ matrix.target }}/release
 
-      - name: FAKE Build Binary ${{ matrix.dir }}/target/${{ matrix.target }}/release/${{ matrix.bin }}
-        if: ${{ false }}
-        run: |
-          mkdir -p ${{ matrix.dir }}/target/${{ matrix.target }}/release
-          touch ${{ matrix.dir }}/target/${{ matrix.target }}/release/${{ matrix.bin }}
+          mkdir -p ${{ env.CACHE_RESULT_DIR }}
+          cp target/${{ matrix.target }}/release/${{ matrix.bin }} ${{ env.CACHE_RESULT_BIN }}
+
+          ls -lA ${{ env.CACHE_RESULT_DIR }}
 
-      - name: Artifact ${{ matrix.name }}
+      - name: Artifact ${{ matrix.result }}
         uses: actions/upload-artifact@v3
         with:
-          path: ${{ matrix.dir }}/target/${{ matrix.target }}/release/${{ matrix.bin }}
-          name: ${{ matrix.name }}--${{ matrix.bin }}
+          path: ${{ env.CACHE_RESULT_BIN }}
+          name: ${{ matrix.result }}--${{ matrix.bin }}
+          if-no-files-found: error
+          retention-days: 3
+
     outputs:
       version: ${{ steps.version.outputs.VERSION }}
 
@@ -172,7 +205,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        name:
+        result:
           - git-remote-gosh-darwin-arm64
           - git-remote-gosh-darwin-amd64
           - git-remote-gosh-linux-arm64
@@ -186,32 +219,36 @@ jobs:
         run: ls -R
 
       ## IMPORTANT! only works with non-windows binaries
-      - name: Gather all binaries "${{ matrix.name }}"
+      - name: Gather all binaries "${{ matrix.result }}"
         run: |
-          mkdir -p ${{ matrix.name }}
-          cp ${{ matrix.name }}--*/* ${{ matrix.name }}
-          ls -lA ${{ matrix.name }}
+          mkdir -p ${{ matrix.result }}
+          cp ${{ matrix.result }}--*/* ${{ matrix.result }}
+          ls -lA ${{ matrix.result }}
 
       - name: Make archive
         run: |
-          chmod +x ${{ matrix.name }}/*
-          tee <<EOF '${{ matrix.name }}/dispatcher.ini'
-          git-remote-gosh_v6_1_0
-          git-remote-gosh_v6_0_0
-          git-remote-gosh_v5_1_0
-          git-remote-gosh_v5_0_0
-          git-remote-gosh_v4_0_0
-          git-remote-gosh_v3_0_0
-          git-remote-gosh_v2_0_0
-          git-remote-gosh_v1_0_0
-          EOF
-
-          tar -czvf ${{ matrix.name }}.tar.gz -C ${{ matrix.name }} .
-
-      - name: Publish
+          chmod +x ${{ matrix.result }}/*
+          printf "%s\n" ${{ matrix.result }}/git-remote-gosh_* | sort -rV | tee '${{ matrix.result }}/dispatcher.ini'
+
+          tar -czvf ${{ matrix.result }}.tar.gz -C ${{ matrix.result }} .
+
+      - name: Upload Draft
         uses: softprops/action-gh-release@v1
+        if: env.release != 'true'
+        with:
+          draft: true
+          tag_name: rc-${{ needs.build.outputs.version || 'no-version' }}-${{ github.ref_name }}-${{ github.run_number }}
+          files: |
+            ${{ matrix.result }}.tar.gz
+          body: ""
+
+      - name: Upload Prerelease
+        uses: softprops/action-gh-release@v1
+        if: env.release == 'true'
         with:
           prerelease: true
+          tag_name: rc-${{ github.ref_name }}-build-${{ github.run_number }}
           files: |
-            ${{ matrix.name }}.tar.gz
-          tag_name: rc-${{ needs.build.outputs.version || 'no-version' }}-${{ github.ref_name }}-${{ github.run_number }}
+            ${{ matrix.result }}.tar.gz
+          name: "Version: ${{ github.ref_name }}"
+          body: ""
diff --git a/gosh-dispatcher/Cargo.toml b/gosh-dispatcher/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "git-remote-gosh"
-version = "6.1.32"
+version = "6.1.33"
 edition = "2021"
 keywords = ['git-remote', 'git', 'gosh']
 rust-version = "1.70"

diff --git a/v6_x/v6.0.0/git-remote-gosh/Cargo.toml b/v6_x/v6.0.0/git-remote-gosh/Cargo.toml
@@ -13,8 +13,8 @@ edition = '2021'
 keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
-rust-version = "1.68"
-version = '6.0.32'
+rust-version = "1.70"
+version = '6.0.33'
 
 [profile.profiling]
 debug = 1

diff --git a/v6_x/v6.1.0/git-remote-gosh/Cargo.toml b/v6_x/v6.1.0/git-remote-gosh/Cargo.toml
@@ -13,8 +13,8 @@ edition = '2021'
 keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
-rust-version = "1.68"
-version = '6.1.32'
+rust-version = "1.70"
+version = '6.1.33'
 
 [profile.profiling]
 debug = 1

diff --git a/version.json b/version.json
@@ -1,3 +1,3 @@
 {
-    "version": "6.1.32"
+    "version": "6.1.33"
 }