From f00d1d88f3b40973a90525a0e5fcecd760a9c7ca Mon Sep 17 00:00:00 2001 From: 0xfourzerofour Date: Mon, 30 Oct 2023 15:52:01 -0400 Subject: [PATCH] chore(release): add release pipeline with cross arch builds --- .github/workflows/docker-release.yaml | 54 ++++++++ .github/workflows/release.yaml | 125 ++++++++++++++++++ .gitignore | 3 + Cargo.toml | 1 + Cross.toml | 2 + Dockerfile | 6 +- Dockerfile.build | 18 +++ Dockerfile.cross | 15 +++ Makefile | 44 ++++++ crates/types/build.rs | 9 -- .../rundler-launcher/docker-compose.yml | 2 +- 11 files changed, 266 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/docker-release.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 Cross.toml create mode 100644 Dockerfile.build create mode 100644 Dockerfile.cross diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml new file mode 100644 index 000000000..72c8def46 --- /dev/null +++ b/.github/workflows/docker-release.yaml @@ -0,0 +1,54 @@ +name: Docker release + +on: + workflow_dispatch: + push: + tags: + - v* + +env: + CARGO_TERM_COLOR: always + DOCKER_IMAGE_NAME: alchemyplatform/rundler + +jobs: + build: + name: build and push + runs-on: ubuntu-22.04 + permissions: + packages: write + contents: read + steps: + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + toolchain: 1.71.0 + + - name: Install toolchain (nightly) + run: rustup toolchain add nightly --component rustfmt --profile minimal + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: alchemyplatform + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker builder + run: | + docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64 + docker buildx create --use --name cross-builder + + - name: Build and push image + run: | + cargo install cross --git https://github.com/cross-rs/cross + export PROFILE=release + sudo -E env "PATH=$PATH" make docker-build-latest diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..e4c0b6a18 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,125 @@ +## This release action is inpired from https://githug.com/paradigmxyz/reth + +name: Release + +on: + push: + tags: + - v* + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-latest + steps: + - name: Extract version + run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + build: + name: build release + strategy: + matrix: + arch: + [ + aarch64-unknown-linux-gnu, + x86_64-unknown-linux-gnu, + x86_64-apple-darwin, + aarch64-apple-darwin, + x86_64-pc-windows-gnu, + ] + include: + - arch: aarch64-unknown-linux-gnu + platform: ubuntu-latest + profile: maxperf + - arch: x86_64-unknown-linux-gnu + platform: ubuntu-latest + profile: maxperf + - arch: x86_64-apple-darwin + platform: macos-latest + profile: maxperf + - arch: aarch64-apple-darwin + platform: macos-latest + profile: maxperf + - arch: x86_64-pc-windows-gnu + platform: ubuntu-latest + profile: maxperf + + runs-on: ${{ matrix.platform }} + needs: extract-version + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Get latest version of stable Rust + run: rustup update stable + - name: Install target + run: rustup target add ${{ matrix.arch }} + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + # ============================== + # Apple M1 SDK setup + # ============================== + + - name: Apple M1 setup + if: ${{ matrix.job.target == 'aarch64-apple-darwin' }} + run: | + echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV + + # ============================== + # Builds + # ============================== + + - name: Build rundler for ${{ matrix.arch }} + run: | + cargo install cross + env PROFILE=${{ matrix.profile }} make build-${{ matrix.arch }} + + - name: Move cross-compiled binary + if: matrix.arch != 'x86_64-pc-windows-gnu' + run: | + mkdir artifacts + mv target/${{ matrix.arch }}/${{ matrix.profile }}/rundler ./artifacts + + - name: Move cross-compiled binary (Windows) + if: matrix.arch == 'x86_64-pc-windows-gnu' + run: | + mkdir artifacts + mv target/${{ matrix.arch }}/${{ matrix.profile }}/rundler.exe ./artifacts + + # ============================== + # Signing + # ============================== + + - name: Configure GPG and create artifacts + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + export GPG_TTY=$(tty) + echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import + cd artifacts + tar -czf rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz rundler* + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + mv *tar.gz* .. + shell: bash + + # ======================================================================= + # Upload artifacts + # This is required to share artifacts between different jobs + # ======================================================================= + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + path: rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + + - name: Upload signature + uses: actions/upload-artifact@v3 + with: + name: rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc + path: rundler-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc diff --git a/.gitignore b/.gitignore index d3b0897d5..728fbc633 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ *.profraw .DS_Store .helix + +# Release artifacts +dist/ diff --git a/Cargo.toml b/Cargo.toml index 8eb8123e7..0c7159b31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,3 +43,4 @@ tower = "0.4.13" tracing = "0.1.37" strum = "0.25.0" url = "2.3.1" + diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 000000000..76c3e817f --- /dev/null +++ b/Cross.toml @@ -0,0 +1,2 @@ +[build] +dockerfile = "Dockerfile.build" diff --git a/Dockerfile b/Dockerfile index 15b72a803..d02b40be2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Adapted from https://github.com/paradigmxyz/reth/blob/main/Dockerfile # syntax=docker/dockerfile:1.4 -FROM rust:1.72.0 AS chef-builder +FROM --platform=$TARGETPLATFORM rust:1.72.0 AS chef-builder # Install system dependencies RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list @@ -50,8 +50,8 @@ WORKDIR /app # install curl for healthcheck RUN apt-get -y update; apt-get -y install curl -# Copy reth over from the build stage +# Copy rundler over from the build stage COPY --from=builder /app/target/release/rundler /usr/local/bin EXPOSE 3000 8080 -CMD ["/usr/local/bin/rundler", "node"] +ENTRYPOINT ["/usr/local/bin/rundler"] diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 000000000..7b8f19e4e --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,18 @@ +ARG CROSS_BASE_IMAGE +# Dockerfile.forge +FROM ghcr.io/foundry-rs/foundry:latest as foundry + +FROM $CROSS_BASE_IMAGE +COPY --from=foundry /usr/local/bin/forge /usr/local/bin/forge + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y gnupg2 apt-transport-https ca-certificates software-properties-common +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN mkdir -p /etc/apt/keyrings +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config protobuf-compiler nodejs yarn +RUN add-apt-repository ppa:ethereum/ethereum +RUN apt-get update +RUN apt-get install -y solc diff --git a/Dockerfile.cross b/Dockerfile.cross new file mode 100644 index 000000000..d665d9c67 --- /dev/null +++ b/Dockerfile.cross @@ -0,0 +1,15 @@ +# This image is meant to enable cross-architecture builds. +# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is +# locatable in `./dist/bin/$TARGETARCH` +FROM --platform=$TARGETPLATFORM ubuntu:22.04 + +LABEL org.opencontainers.image.source=https://github.com/alchemyplatform/rundler +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" + +# Filled by docker buildx +ARG TARGETARCH + +COPY ./dist/bin/$TARGETARCH/rundler /usr/local/bin/rundler + +EXPOSE 3000 8080 +ENTRYPOINT ["/usr/local/bin/rundler"] diff --git a/Makefile b/Makefile index 6b3339083..fe28c722a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,11 @@ ##@ Test UNIT_TEST_ARGS := --locked --workspace --all-features +PROFILE ?= release +DOCKER_IMAGE_NAME ?= alchemyplatform/rundler +BIN_DIR = "dist/bin" +BUILD_PATH = "target" +GIT_TAG ?= $(shell git describe --tags --abbrev=0) .PHONY: build build: ## Build the project. @@ -28,4 +33,43 @@ test-spec-integrated: ## Run spec tests in integrated mode .PHONY: test-spec-modular test-spec-modular: ## Run spec tests in modular mode test/spec-tests/remote/run-spec-tests.sh + +.PHONY: submodule-update +submodule-update: ## Run spec tests in modular mode + git submodule update + +# Note: The additional rustc compiler flags are for intrinsics needed by MDBX. +# See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std +build-%: + cross build --bin rundler --target $* --profile "$(PROFILE)" +# Note: This requires a buildx builder with emulation support. For example: +.PHONY: docker-build-latest +docker-build-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`. + $(call build_docker_image,$(GIT_TAG),latest) + +# Create a cross-arch Docker image with the given tags and push it +define build_docker_image + $(MAKE) build-aarch64-unknown-linux-gnu + mkdir -p $(BIN_DIR)/arm64 + cp $(BUILD_PATH)/aarch64-unknown-linux-gnu/$(PROFILE)/rundler $(BIN_DIR)/arm64/rundler + + $(MAKE) build-x86_64-unknown-linux-gnu + mkdir -p $(BIN_DIR)/amd64 + cp $(BUILD_PATH)/x86_64-unknown-linux-gnu/$(PROFILE)/rundler $(BIN_DIR)/amd64/rundler + + docker buildx build --file ./Dockerfile.cross . \ + --platform linux/arm64,linux/amd64 \ + --tag $(DOCKER_IMAGE_NAME):$(1) \ + --tag $(DOCKER_IMAGE_NAME):$(2) \ + --provenance=false +endef + + + +# docker buildx build --file ./Dockerfile.cross . \ +# --platform linux/amd64,linux/arm64 \ +# --tag $(DOCKER_IMAGE_NAME):$(1) \ +# --tag $(DOCKER_IMAGE_NAME):$(2) \ +# --provenance=false \ +# --push diff --git a/crates/types/build.rs b/crates/types/build.rs index 6b124e65b..0182434d3 100644 --- a/crates/types/build.rs +++ b/crates/types/build.rs @@ -19,7 +19,6 @@ fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=contracts/lib"); println!("cargo:rerun-if-changed=contracts/src"); println!("cargo:rerun-if-changed=contracts/foundry.toml"); - update_submodules()?; generate_contract_bindings()?; Ok(()) } @@ -62,14 +61,6 @@ fn generate_abis() -> Result<(), Box> { ) } -fn update_submodules() -> Result<(), Box> { - run_command( - Command::new("git").arg("submodule").arg("update"), - "https://github.com/git-guides/install-git", - "update submodules", - ) -} - fn run_command( command: &mut Command, install_page_url: &str, diff --git a/test/spec-tests/launchers/rundler-launcher/docker-compose.yml b/test/spec-tests/launchers/rundler-launcher/docker-compose.yml index 09f931a59..9032037a9 100644 --- a/test/spec-tests/launchers/rundler-launcher/docker-compose.yml +++ b/test/spec-tests/launchers/rundler-launcher/docker-compose.yml @@ -6,7 +6,7 @@ services: ports: - "3000:3000" - "8080:8080" - command: bash -c "/usr/local/bin/rundler node" + command: node environment: - RUST_LOG=debug - ENTRY_POINTS=0x5FF137D4b0FDCD49DcA30c7CF57E578a026d2789