feat(CI): optimize rust deps for Docker CI (#1278)

Co-authored-by: Klaus Lungwitz <[email protected]> Co-authored-by: Mariano Nicolini <[email protected]> Co-authored-by: Klaus @ LambdaClass <[email protected]> Co-authored-by: JuArce <[email protected]>
yetanotherco · Nov 4, 2024 · c980252 · c980252
1 parent d5d8eb3
commit c980252
Show file tree

Hide file tree

Showing 7 changed files with 158 additions and 28 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -11,4 +11,4 @@ volume
 nonce_*.bin
 docker-compose.yaml
 .github/**
-**.md
+**.md
diff --git a/.github/workflows/send-proofs-docker.yml b/.github/workflows/send-proofs-docker.yml
@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Log in to GitHub Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.actor }}

diff --git a/config-files/config-aggregator-docker.yaml b/config-files/config-aggregator-docker.yaml
@@ -29,4 +29,4 @@ aggregator:
   telemetry_ip_port_address: localhost:4001
   garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days)
   garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days)
-  garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours)
+  garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours)
diff --git a/docker/aggregator.Dockerfile b/docker/aggregator.Dockerfile
@@ -9,6 +9,7 @@ COPY core         ./core
 COPY metrics      ./metrics
 COPY contracts/bindings/ ./contracts/bindings
 
+RUN go get github.com/ethereum/go-ethereum@latest
 RUN go build -o ./aligned-layer-aggregator aggregator/cmd/main.go
 
 FROM debian:bookworm-slim

diff --git a/docker/aligned_base.Dockerfile b/docker/aligned_base.Dockerfile
@@ -1,7 +1,4 @@
-FROM rust:slim-bookworm
-
-# Install rust nightly-2024-04-17
-RUN rustup toolchain install nightly-2024-04-17
+FROM debian:bookworm-slim AS base
 
 ARG BUILDARCH
 ENV GO_VERSION=1.22.2
@@ -31,12 +28,121 @@ RUN go install github.com/maoueh/zap-pretty@latest
 RUN go install github.com/ethereum/go-ethereum/cmd/abigen@latest
 RUN go install github.com/Layr-Labs/eigenlayer-cli/cmd/eigenlayer@latest
 
+# Install rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
 WORKDIR /aligned_layer
 
 COPY Makefile .
-COPY operator ./operator
-COPY batcher/aligned-sdk ./batcher/aligned-sdk
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+
+FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
+
+FROM chef AS planner
+
+# build_sp1_linux
+COPY operator/sp1/lib/Cargo.toml /aligned_layer/operator/sp1/lib/Cargo.toml
+COPY operator/sp1/lib/src/ /aligned_layer/operator/sp1/lib/src/
+WORKDIR /aligned_layer/operator/sp1/lib
+RUN cargo chef prepare --recipe-path /aligned_layer/operator/sp1/lib/recipe.json
+
+# build_risc_zero_linux
+COPY operator/risc_zero/lib/Cargo.toml /aligned_layer/operator/risc_zero/lib/Cargo.toml
+COPY operator/risc_zero/lib/src/ /aligned_layer/operator/risc_zero/lib/src/
+WORKDIR /aligned_layer/operator/risc_zero/lib
+RUN cargo chef prepare --recipe-path /aligned_layer/operator/risc_zero/lib/recipe.json
+
+# build_sp1_linux_old
+COPY operator/sp1_old/lib/Cargo.toml /aligned_layer/operator/sp1_old/lib/Cargo.toml
+COPY operator/sp1_old/lib/src/ /aligned_layer/operator/sp1_old/lib/src/
+WORKDIR /aligned_layer/operator/sp1_old/lib/src/
+RUN cargo chef prepare --recipe-path /aligned_layer/operator/sp1_old/lib/recipe.json
+
+# build_risc_zero_linux_old
+COPY operator/risc_zero_old/lib/Cargo.toml /aligned_layer/operator/risc_zero_old/lib/Cargo.toml
+COPY operator/risc_zero_old/lib/src/ /aligned_layer/operator/risc_zero_old/lib/src/
+WORKDIR /aligned_layer/operator/risc_zero_old/lib/src/
+RUN cargo chef prepare --recipe-path /aligned_layer/operator/risc_zero_old/lib/recipe.json
+
+# build_merkle_tree_linux
+COPY operator/merkle_tree/lib/Cargo.toml /aligned_layer/operator/merkle_tree/lib/Cargo.toml
+COPY operator/merkle_tree/lib/src/ /aligned_layer/operator/merkle_tree/lib/src/
+WORKDIR operator/merkle_tree/lib
+RUN cargo chef prepare --recipe-path /aligned_layer/operator/merkle_tree/lib/recipe.json
+
+FROM chef AS chef_builder
+
+COPY batcher/aligned-sdk /aligned_layer/batcher/aligned-sdk/
 
+# build_sp1_linux
+COPY operator/sp1/ /aligned_layer/operator/sp1/
+COPY --from=planner /aligned_layer/operator/sp1/lib/recipe.json /aligned_layer/operator/sp1/lib/recipe.json
+WORKDIR /aligned_layer/operator/sp1/lib/
+RUN cargo chef cook --release --recipe-path /aligned_layer/operator/sp1/lib/recipe.json
+
+# build_risc_zero_linux
+COPY operator/risc_zero/ /aligned_layer/operator/risc_zero/
+COPY --from=planner /aligned_layer/operator/risc_zero/lib/recipe.json /aligned_layer/operator/risc_zero/lib/recipe.json
+WORKDIR /aligned_layer/operator/risc_zero/lib/
+RUN cargo chef cook --release --recipe-path /aligned_layer/operator/risc_zero/lib/recipe.json
+
+# build_sp1_linux_old
+COPY operator/sp1_old/ /aligned_layer/operator/sp1_old/
+COPY --from=planner /aligned_layer/operator/sp1_old/lib/recipe.json /aligned_layer/operator/sp1_old/lib/recipe.json
+WORKDIR /aligned_layer/operator/sp1_old/lib/
+RUN cargo chef cook --release --recipe-path /aligned_layer/operator/sp1_old/lib/recipe.json
+
+# build_risc_zero_linux_old
+COPY operator/risc_zero_old/ /aligned_layer/operator/risc_zero_old/
+COPY --from=planner /aligned_layer/operator/risc_zero_old/lib/recipe.json /aligned_layer/operator/risc_zero_old/lib/recipe.json
+WORKDIR /aligned_layer/operator/risc_zero_old/lib/
+RUN cargo chef cook --release --recipe-path /aligned_layer/operator/risc_zero_old/lib/recipe.json
+
+# build_merkle_tree_linux
+COPY operator/merkle_tree/ /aligned_layer/operator/merkle_tree/
+COPY --from=planner /aligned_layer/operator/merkle_tree/lib/recipe.json /aligned_layer/operator/merkle_tree/lib/recipe.json
+WORKDIR /aligned_layer/operator/merkle_tree/lib/
+RUN cargo chef cook --release --recipe-path /aligned_layer/operator/merkle_tree/lib/recipe.json
+
+FROM base AS builder
+
+ENV RELEASE_FLAG=--release
+ENV TARGET_REL_PATH=release
 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 
-RUN make build_all_ffi_linux
+COPY operator/ /aligned_layer/operator/
+COPY batcher/ /aligned_layer/batcher/
+COPY --from=chef_builder /aligned_layer/batcher/aligned-sdk /aligned_layer/batcher/aligned-sdk
+
+# build_sp1_linux
+COPY --from=chef_builder /aligned_layer/operator/sp1/lib/target/ /aligned_layer/operator/sp1/lib/target/
+WORKDIR /aligned_layer/operator/sp1/lib
+RUN cargo build ${RELEASE_FLAG}
+RUN cp /aligned_layer/operator/sp1/lib/target/${TARGET_REL_PATH}/libsp1_verifier_ffi.so /aligned_layer/operator/sp1/lib/libsp1_verifier_ffi.so
+
+# build_risc_zero_linux
+COPY --from=chef_builder /aligned_layer/operator/risc_zero/lib/target/ /aligned_layer/operator/risc_zero/lib/target/
+WORKDIR /aligned_layer/operator/risc_zero/lib
+RUN cargo build ${RELEASE_FLAG}
+RUN cp /aligned_layer/operator/risc_zero/lib/target/${TARGET_REL_PATH}/librisc_zero_verifier_ffi.so /aligned_layer/operator/risc_zero/lib/librisc_zero_verifier_ffi.so
+
+# build_sp1_linux_old
+COPY --from=chef_builder /aligned_layer/operator/sp1_old/lib/target/ /aligned_layer/operator/sp1_old/lib/target/
+WORKDIR /aligned_layer/operator/sp1_old/lib
+RUN cargo build ${RELEASE_FLAG}
+RUN cp /aligned_layer/operator/sp1_old/lib/target/${TARGET_REL_PATH}/libsp1_verifier_old_ffi.so /aligned_layer/operator/sp1_old/lib/libsp1_verifier_old_ffi.so
+
+# build_risc_zero_linux_old
+COPY --from=chef_builder /aligned_layer/operator/risc_zero_old/lib/target/ /aligned_layer/operator/risc_zero_old/lib/target/
+WORKDIR /aligned_layer/operator/risc_zero_old/lib
+RUN cargo build ${RELEASE_FLAG}
+RUN cp /aligned_layer/operator/risc_zero_old/lib/target/${TARGET_REL_PATH}/librisc_zero_verifier_old_ffi.so /aligned_layer/operator/risc_zero_old/lib/librisc_zero_verifier_old_ffi.so
+
+
+# build_merkle_tree_linux
+COPY --from=chef_builder /aligned_layer/operator/merkle_tree/lib/target/ /aligned_layer/operator/merkle_tree/lib/target/
+WORKDIR /aligned_layer/operator/merkle_tree/lib
+RUN cargo build ${RELEASE_FLAG}
+RUN cp /aligned_layer/operator/merkle_tree/lib/target/${TARGET_REL_PATH}/libmerkle_tree.so /aligned_layer/operator/merkle_tree/lib/libmerkle_tree.so
+RUN cp /aligned_layer/operator/merkle_tree/lib/target/${TARGET_REL_PATH}/libmerkle_tree.a /aligned_layer/operator/merkle_tree/lib/libmerkle_tree.a
diff --git a/docker/batcher.Dockerfile b/docker/batcher.Dockerfile
@@ -1,33 +1,55 @@
-FROM ghcr.io/yetanotherco/aligned_layer/aligned_base:latest AS builder
-
-RUN apt update -y
-RUN apt install -y gcc
+FROM ghcr.io/yetanotherco/aligned_layer/aligned_base:latest AS base
 
 COPY go.mod .
 COPY go.sum .
-COPY batcher ./batcher
+COPY batcher/aligned-batcher/gnark/verifier.go /aligned_layer/batcher/aligned-batcher/gnark/verifier.go
 
-WORKDIR /aligned_layer/batcher/aligned-batcher
+RUN apt update -y && apt install -y gcc
+RUN go build -buildmode=c-archive -o libverifier.a /aligned_layer/batcher/aligned-batcher/gnark/verifier.go
 
-RUN go build -buildmode=c-archive -o libverifier.a ./gnark/verifier.go
+FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
 
-WORKDIR /aligned_layer
+FROM chef AS planner
 
-COPY batcher/aligned-batcher/Cargo.toml batcher/aligned-batcher/Cargo.toml
-COPY batcher/aligned/Cargo.toml batcher/aligned/Cargo.toml
+COPY batcher/aligned-batcher/Cargo.toml /aligned_layer/batcher/aligned-batcher/Cargo.toml
+COPY batcher/aligned-batcher/src/main.rs /aligned_layer/batcher/aligned-batcher/src/main.rs
+WORKDIR /aligned_layer/batcher/aligned-batcher/
+RUN cargo chef prepare --recipe-path /aligned_layer/batcher/aligned-batcher/recipe.json
 
-RUN cargo build --manifest-path ./batcher/aligned-batcher/Cargo.toml --release
-RUN cargo build --manifest-path ./batcher/aligned/Cargo.toml --release
+COPY batcher/aligned/Cargo.toml /aligned_layer/batcher/aligned/Cargo.toml
+COPY batcher/aligned/src/main.rs /aligned_layer/batcher/aligned/src/main.rs
+WORKDIR /aligned_layer/batcher/aligned/
+RUN cargo chef prepare --recipe-path /aligned_layer/batcher/aligned/recipe.json
 
-COPY scripts/test_files/gnark_groth16_bn254_infinite_script scripts/test_files/gnark_groth16_bn254_infinite_script
+FROM chef AS chef_builder
+COPY batcher/aligned-sdk/ /aligned_layer/batcher/aligned-sdk/
 
-RUN go build -o ./gnark_groth16_bn254_infinite_script scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go
+COPY --from=planner /aligned_layer/batcher/aligned-batcher/recipe.json /aligned_layer/batcher/aligned-batcher/recipe.json
+WORKDIR /aligned_layer/batcher/aligned-batcher
+RUN cargo chef cook --release --recipe-path /aligned_layer/batcher/aligned-batcher/recipe.json
 
-RUN rm -rf operator/
+COPY --from=planner /aligned_layer/batcher/aligned/recipe.json /aligned_layer/batcher/aligned/recipe.json
+WORKDIR /aligned_layer/batcher/aligned/
+RUN cargo chef cook --release --recipe-path /aligned_layer/batcher/aligned/recipe.json
 
-FROM debian:bookworm-slim
+FROM base AS builder
+COPY . /aligned_layer/
 
+COPY --from=chef_builder /aligned_layer/batcher/aligned-batcher/target/ /aligned_layer/batcher/aligned-batcher/target/
+WORKDIR /aligned_layer/batcher/aligned-batcher/
+RUN cargo build --manifest-path /aligned_layer/batcher/aligned-batcher/Cargo.toml --release
+
+COPY --from=chef_builder /aligned_layer/batcher/aligned/target/ /aligned_layer/batcher/aligned/target/
+WORKDIR /aligned_layer/batcher/aligned/
+RUN cargo build --manifest-path /aligned_layer/batcher/aligned/Cargo.toml --release
+
+COPY scripts/test_files/gnark_groth16_bn254_infinite_script/ /aligned_layer/scripts/test_files/gnark_groth16_bn254_infinite_script/
 WORKDIR /aligned_layer
+RUN go build -o /aligned_layer/gnark_groth16_bn254_infinite_script scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go
+
+RUN rm -rf operator/
+
+FROM debian:bookworm-slim AS final
 
 COPY --from=builder /aligned_layer /aligned_layer
 COPY --from=builder /aligned_layer/batcher/target/release/aligned-batcher /usr/local/bin/
@@ -38,7 +60,6 @@ COPY ../scripts/test_files/ ./scripts/test_files
 COPY ./config-files/config-batcher-docker.yaml ./config-files/
 COPY ./config-files/anvil.batcher.ecdsa.key.json ./config-files/
 
-RUN apt update -y
-RUN apt install -y libssl-dev ca-certificates
+RUN apt update -y && apt install -y libssl-dev ca-certificates
 
 CMD ["aligned-batcher", "--config", "./config-files/config-batcher-docker.yaml"]
diff --git a/docker/operator.Dockerfile b/docker/operator.Dockerfile
@@ -1,5 +1,7 @@
 FROM ghcr.io/yetanotherco/aligned_layer/aligned_base:latest AS builder
 
+WORKDIR /aligned_layer
+
 RUN apt update -y
 RUN apt install -y gcc