From 712c60e6fcda1e18f3882012afd6a7f62b25a2bd Mon Sep 17 00:00:00 2001 From: Tiram <18632023+tiram88@users.noreply.github.com> Date: Sat, 30 Sep 2023 21:26:45 +0200 Subject: [PATCH] Fix the CI build for the Ubuntu release (#287) * Fix the CI build for the Ubuntu release * Add reference * Restrict openssl dependency to linux target * Use arduino setup-protoc action in all CI jobs * Add some file cleaning to test CI for not going out of disk space * Prevent llvm removal in test CI * Robust removal * Add a job checking successful build of Ubuntu release --- .github/workflows/ci.yaml | 109 ++++++++++++++++++++++++++++++++----- Cargo.lock | 11 ++++ rpc/wrpc/server/Cargo.toml | 6 ++ 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4edf8b0e5..707456a31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,8 +10,10 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - - name: Install protoc - run: sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install stable toolchain uses: dtolnay/rust-toolchain@stable @@ -39,6 +41,44 @@ jobs: # matrix: # os: [ ubuntu-latest, macos-latest, windows-latest ] steps: + # Clean unnecessary files to save disk space + - name: clean unnecessary files to save space + run: | + docker rmi `docker images -q` + sudo rm -rf /usr/share/dotnet /etc/mysql /etc/php /etc/sudo apt/sources.list.d + sudo apt -y autoremove --purge + sudo apt -y autoclean + sudo apt clean + rm --recursive --force "$AGENT_TOOLSDIRECTORY" + df -h + + # remove large packages manually (all but llvm) + sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..." + sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..." + sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..." + sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..." + df -h + + # Free up disk space on Ubuntu + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # This might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: false + + # large packages, except llvm, are removed manually during the previous step + # see: https://github.com/jlumbroso/free-disk-space/issues/6 + # TODO: use the discussed whitelist feature when available + large-packages: false + + swap-storage: true + - name: Checkout sources uses: actions/checkout@v3 @@ -46,8 +86,10 @@ jobs: if: runner.os == 'Windows' run: git config --global core.autocrlf false - - name: Install protoc - run: sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install stable toolchain uses: dtolnay/rust-toolchain@stable @@ -129,8 +171,10 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - - name: Install protoc - run: sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install stable toolchain uses: dtolnay/rust-toolchain@stable @@ -162,10 +206,10 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - - name: Install protoc - run: | - sudo apt update -y - sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install llvm id: install_llvm @@ -223,10 +267,10 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - - name: Install protoc - run: | - sudo apt update -y - sudo apt install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install llvm id: install_llvm @@ -271,3 +315,40 @@ jobs: - name: Build wasm pack for nodejs run: cd wasm && wasm-pack build --target nodejs --out-dir nodejs/kaspa --features full + + build-release: + name: Build Ubuntu Release + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install zig + if: runner.os == 'Linux' + uses: goto-bus-stop/setup-zig@v2 # needed for cargo-zigbuild + + - name: Build on Linux + if: runner.os == 'Linux' + # We're using musl to make the binaries statically linked and portable + run: | + cargo install cargo-zigbuild + cargo --verbose zigbuild --bin kaspad --bin simpa --bin rothschild --release --target x86_64-unknown-linux-gnu.2.27 # Use an older glibc version diff --git a/Cargo.lock b/Cargo.lock index 8dd043287..430e2b7e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3030,6 +3030,7 @@ dependencies = [ "kaspa-wrpc-core", "log", "num_cpus", + "openssl", "paste", "serde", "thiserror", @@ -3702,6 +3703,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.28.0+1.1.1w" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ce95ee1f6f999dfb95b8afd43ebe442758ea2104d1ccb99a94c30db22ae701f" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.90" @@ -3710,6 +3720,7 @@ checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/rpc/wrpc/server/Cargo.toml b/rpc/wrpc/server/Cargo.toml index e13f4e0e9..b1f803467 100644 --- a/rpc/wrpc/server/Cargo.toml +++ b/rpc/wrpc/server/Cargo.toml @@ -33,3 +33,9 @@ workflow-core.workspace = true workflow-log.workspace = true workflow-rpc.workspace = true num_cpus.workspace = true + +[target.x86_64-unknown-linux-gnu.dependencies] +# Adding explicitely the openssl dependency here is needed for a successful build with zigbuild +# as used in the release deployment in GitHub CI +# see: https://github.com/rust-cross/cargo-zigbuild/issues/127 +openssl = { version = "0.10", features = ["vendored"] }