From 7960bfce8c442e5c5e8aa22e34ab3b1a122f0e87 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 20 Feb 2023 23:09:02 -0800 Subject: [PATCH] ci: Add workflows. (#1) * Add CI flow. * Update CI. * Add release flow. * Add dockerfile. * Fix path. * Add config folder. * Fix path. * Disable for now. --- .cargo/config.toml | 5 ++ .cargo/nextest.toml | 4 + .github/workflows/ci.yml | 72 +++++++++++++++ .github/workflows/release.yml | 165 ++++++++++++++++++++++++++++++++++ depot.json | 1 + release/Dockerfile | 91 +++++++++++++++++++ 6 files changed, 338 insertions(+) create mode 100644 .cargo/config.toml create mode 100644 .cargo/nextest.toml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 depot.json create mode 100644 release/Dockerfile diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..1cf2a18c2 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[profile.release] +codegen-units = 1 +lto = true +panic = "abort" +strip = "debuginfo" diff --git a/.cargo/nextest.toml b/.cargo/nextest.toml new file mode 100644 index 000000000..cc86ee4ad --- /dev/null +++ b/.cargo/nextest.toml @@ -0,0 +1,4 @@ +[profile.ci] +retries = 1 +test-threads = 1 +fail-fast = false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..bd04d0033 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +jobs: + format: + name: Format + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + name: Setup toolchain + with: + toolchain: 1.67.0 + profile: minimal + components: rustfmt + - uses: actions-rs/cargo@v1 + name: Check formatting + with: + command: fmt + args: --all --check + lint: + name: Lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + name: Setup toolchain + with: + toolchain: 1.67.0 + profile: minimal + components: clippy + - uses: actions-rs/cargo@v1 + name: Run linter + with: + command: clippy + args: --workspace --all-targets + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + name: Setup toolchain + with: + toolchain: 1.67.0 + profile: minimal + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + shared-key: ci + - uses: actions-rs/cargo@v1 + name: Run tests + with: + command: test + args: --workspace diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..4eb24e405 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,165 @@ +name: Release + +on: + # Manually release crates + workflow_dispatch: + # Test on master to ensure PRs are good + # push: + # branches: + # - master + # - develop-* + # - release-* + # Uncomment to test in PRs (its safe) + # pull_request: + +permissions: + contents: write + id-token: write + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + host: ubuntu-20.04 + binary: proto + docker-target: bin-gnu + docker-platform: linux/amd64 + + - target: x86_64-unknown-linux-musl + host: ubuntu-20.04 + binary: proto + docker-target: bin-musl + docker-platform: linux/amd64 + + - target: aarch64-unknown-linux-gnu + host: ubuntu-20.04 + binary: proto + docker-target: bin-gnu + docker-platform: linux/arm64 + + - target: aarch64-unknown-linux-musl + host: ubuntu-20.04 + binary: proto + docker-target: bin-musl + docker-platform: linux/arm64 + + - target: x86_64-apple-darwin + host: macos-latest + binary: proto + setup: | + export MACOSX_DEPLOYMENT_TARGET="10.13"; + + - target: aarch64-apple-darwin + host: macos-12 + binary: proto + setup: | + export CC=$(xcrun -f clang); + export CXX=$(xcrun -f clang++); + export SDKROOT=$(xcrun -sdk macosx --show-sdk-path); + export CFLAGS="-isysroot $SDKROOT -isystem $SDKROOT"; + export MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version); + + - target: x86_64-pc-windows-msvc + host: windows-2022 + binary: proto.exe + name: Stable - ${{ matrix.target }} + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + if: ${{ !matrix.docker-target }} + with: + override: true + profile: minimal + target: ${{ matrix.target }} + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: release + - name: Generate lockfile + uses: actions-rs/cargo@v1 + if: ${{ !matrix.docker-target }} + with: + command: generate-lockfile + - name: Setup toolchain + if: ${{ matrix.setup }} + run: ${{ matrix.setup }} + - name: Add target + if: ${{ !matrix.docker-target }} + run: rustup target add ${{ matrix.target }} + - name: Build binary + uses: actions-rs/cargo@v1 + if: ${{ !matrix.docker-target }} + with: + command: build + args: --release --target ${{ matrix.target }} + - name: Install Depot CLI + if: ${{ matrix.docker-target }} + uses: depot/setup-action@v1 + - name: Build binary with Docker / Depot + if: ${{ matrix.docker-target }} + uses: depot/build-push-action@v1 + with: + context: . + file: ./release/Dockerfile + target: ${{ matrix.docker-target }} + platforms: ${{ matrix.docker-platform }} + outputs: type=local,dest=. + - uses: actions/upload-artifact@v2 + name: Upload artifact + with: + name: binary-${{ matrix.target }} + path: ./target/${{ matrix.target }}/release/${{ matrix.binary }} + if-no-files-found: error + + test: + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + host: ubuntu-20.04 + + - target: x86_64-unknown-linux-musl + host: ubuntu-20.04 + image: clux/muslrust:stable + + - target: x86_64-apple-darwin + host: macos-latest + + - target: x86_64-pc-windows-msvc + host: windows-latest + needs: + - build + name: Test - ${{ matrix.target }} + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: + shared-key: proto + - name: Setup toolchain + if: ${{ matrix.setup }} + run: ${{ matrix.setup }} + - uses: actions/download-artifact@v2 + name: Download artifacts + with: + path: artifacts + - name: List packages + run: ls -lR ./artifacts + shell: bash + - name: Test binary + if: ${{ !matrix.image }} + run: bash ./scripts/release/testBinary.sh + env: + TARGET: ${{ matrix.target }} + - name: Test binary with Docker + if: ${{ matrix.image }} + uses: mosteo-actions/docker-run@v1 + with: + image: ${{ matrix.image }} + params: -e TARGET=${{ matrix.target }} + command: bash ./scripts/release/testBinary.sh diff --git a/depot.json b/depot.json new file mode 100644 index 000000000..e87bc8b64 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{ "id": "7p4m38rwqh" } diff --git a/release/Dockerfile b/release/Dockerfile new file mode 100644 index 000000000..f78860e9e --- /dev/null +++ b/release/Dockerfile @@ -0,0 +1,91 @@ +# NOTE: this Dockerfile is primarily used to compile proto's linux binaries. +# It supports compiling x86_64 and aarch64 binaries for both glibc and musl. +# +# Usage: docker buildx build --output type=local,dest=./output --platform linux/amd64,linux/arm64 -f scripts/Dockerfile . + +# glibc + +FROM rust:1.67.0 AS base-gnu +RUN cargo install cargo-chef --version ^0.1 +WORKDIR /app +COPY .cargo .cargo +COPY rust-toolchain.toml . +RUN rustup update + +FROM base-gnu AS plan-gnu +COPY Cargo* . +COPY crates crates +RUN cargo chef prepare --recipe-path recipe.json + +FROM base-gnu AS build-gnu +ENV RUSTFLAGS "-C strip=symbols" +COPY --from=plan-gnu /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json +COPY Cargo* . +COPY crates crates +RUN cargo build --bin proto --release +RUN set -ex; \ + export target="$(rustc --version --verbose | grep 'host:' | cut -d' ' -f2)"; \ + mkdir -p "/out/target/$target/release" && cp target/release/proto "/out/target/$target/release/proto" + +FROM scratch AS bin-gnu +COPY --from=build-gnu /out/target/ /target/ + +# musl (amd64) + +FROM messense/rust-musl-cross:x86_64-musl AS base-musl-amd64 +RUN cargo install cargo-chef --version ^0.1 +WORKDIR /app +COPY rust-toolchain.toml . +RUN rustup update && rustup target add x86_64-unknown-linux-musl + +FROM base-musl-amd64 AS plan-musl-amd64 +COPY Cargo* . +COPY crates crates +RUN cargo chef prepare --recipe-path recipe.json + +FROM base-musl-amd64 AS build-musl-amd64 +ENV RUSTFLAGS "-C strip=symbols" +COPY --from=plan-musl-amd64 /app/recipe.json recipe.json +RUN cargo chef cook --release --target x86_64-unknown-linux-musl --recipe-path recipe.json +COPY Cargo* . +COPY crates crates +RUN cargo build --bin proto --release --target x86_64-unknown-linux-musl +RUN mkdir -p /out && cp --parents target/*/release/proto /out/ + +# musl (arm64) + +FROM messense/rust-musl-cross:aarch64-musl AS base-musl-arm64 +RUN cargo install cargo-chef --version ^0.1 +WORKDIR /app +COPY rust-toolchain.toml . +RUN rustup update && rustup target add aarch64-unknown-linux-musl + +FROM base-musl-arm64 AS plan-musl-arm64 +COPY Cargo* . +COPY crates crates +RUN cargo chef prepare --recipe-path recipe.json + +FROM base-musl-arm64 AS build-musl-arm64 +ENV RUSTFLAGS "-C strip=symbols" +COPY --from=plan-musl-arm64 /app/recipe.json recipe.json +RUN cargo chef cook --release --target aarch64-unknown-linux-musl --recipe-path recipe.json +COPY Cargo* . +COPY crates crates +RUN cargo build --bin proto --release --target aarch64-unknown-linux-musl +RUN mkdir -p /out && cp --parents target/*/release/proto /out/ + +# musl (dynamic arch based on target) + +FROM build-musl-${TARGETARCH} AS build-musl + +FROM scratch AS bin-musl +COPY --from=build-musl /out/target/ /target/ + +# combined glibc and musl + +FROM scratch AS bin +COPY --from=bin-gnu /target/ /target/ +COPY --from=bin-musl /target/ /target/ + +FROM bin