From 4a33621c82a10cdfffc0a2699d0450ad1f061a06 Mon Sep 17 00:00:00 2001 From: Leeren Chang Date: Fri, 16 Aug 2024 15:20:32 -0700 Subject: [PATCH] support multi-arch build --- .github/workflows/ci-geth-s3.yml | 157 ++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci-geth-s3.yml b/.github/workflows/ci-geth-s3.yml index 6bcb24058d61..022c386df84a 100644 --- a/.github/workflows/ci-geth-s3.yml +++ b/.github/workflows/ci-geth-s3.yml @@ -3,8 +3,7 @@ name: Build and Upload geth Binary on: push: branches: - - main - workflow_dispatch: + - feat/main permissions: id-token: write @@ -13,24 +12,25 @@ permissions: actions: write env: - NUM_BINARIES_TO_KEEP: 50 + NUM_INTERNAL_BINARIES_TO_KEEP: 51 + NUM_PUBLIC_BINARIES_TO_KEEP: 400 jobs: # Add timestamp Timestamp: - # limit a few authorized users to trigger this job - # since all other jobs depend on this job, no need to secure other jobs - if: github.actor == 'andybowu' || github.actor == 'edisonz0718' || github == 'LeoHChen' uses: storyprotocol/gha-workflows/.github/workflows/reusable-timestamp.yml@main # Build and upload the geth binary build_and_push: needs: Timestamp runs-on: ubuntu-latest + strategy: + matrix: + platform: [linux-386, linux-amd64, linux-arm, linux-arm64, darwin-amd64, darwin-arm64, windows-amd64, windows-386] steps: - name: Checkout code - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + uses: actions/checkout@v4.1.5 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -39,39 +39,128 @@ jobs: aws-region: us-west-1 role-session-name: github-actions + - name: Extract the version + run: | + PARAMS_FILE="./params/version.go" + VERSION_MAJOR=$(awk -F= '/VersionMajor/ {gsub(/[^0-9]/, "", $2); printf "%s", $2}' $PARAMS_FILE) + VERSION_MINOR=$(awk -F= '/VersionMinor/ {gsub(/[^0-9]/, "", $2); printf "%s", $2}' $PARAMS_FILE) + VERSION_PATCH=$(awk -F= '/VersionPatch/ {gsub(/[^0-9]/, "", $2); printf "%s", $2}' $PARAMS_FILE) + VERSION_META=$(awk -F\" '/VersionMeta/ {print $2; exit}' $PARAMS_FILE) + + # Construct the full version string + VERSION="$VERSION_MAJOR.$VERSION_MINOR.$VERSION_PATCH" + if [ "$VERSION_META" != "stable" ]; then + VERSION+="-${VERSION_META}" + fi + + echo "Version extracted: $VERSION" + echo "VERSION=$VERSION" >> $GITHUB_ENV + - name: Build the geth binary run: | - make geth + IFS="-" read -r GOOS GOARCH <<< "${{ matrix.platform }}" + output_name=./build/bin/geth + if [ "$GOOS" = "windows" ]; then + output_name+='.exe' + fi + + echo "Building for $GOOS/$GOARCH..." + env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name ./cmd/geth + + if [ $? -ne 0 ]; then + echo "Build failed!" + exit 1 + fi + + if [ ! -f "$output_name" ]; then + echo "Geth binary not found!" + exit 1 + fi + chmod +x "$output_name" - name: Upload the geth binary to S3 run: | export TZ=America/Los_Angeles - VERSION=$(date +%Y%m%d%H%M%S) + IFS="-" read -r GOOS GOARCH <<< "${{ matrix.platform }}" + TIMESTAMP=$(date +%Y%m%d%H%M%S) HUMAN_READABLE_VERSION=$(date) - echo "Building version $VERSION at $HUMAN_READABLE_VERSION" - aws s3 cp ./build/bin/geth s3://iliad-geth-binaries/geth/geth-$VERSION --quiet - - # Update manifest file - aws s3 cp s3://iliad-geth-binaries/geth/manifest.txt manifest.txt --quiet || touch manifest.txt - echo $VERSION >> manifest.txt - aws s3 cp manifest.txt s3://iliad-geth-binaries/geth/manifest.txt --quiet - - echo "ILIAD_VERSION=$VERSION" >> $GITHUB_ENV - + COMMIT_HASH=$(git rev-parse --short HEAD) + FOLDER_NAME="geth-${{ matrix.platform }}-${VERSION}-${COMMIT_HASH}" + ARCHIVE_NAME="${FOLDER_NAME}.tar.gz" + + binary_name=./build/bin/geth + if [ "$GOOS" = "windows" ]; then + binary_name+='.exe' + fi + + # For linux amd64 upload the binary for internal testing + if [ "${{ matrix.platform }}" = "linux-amd64" ]; then + + echo "Uploading binary for internal use..." + aws s3 cp $binary_name s3://iliad-geth-binaries/geth/geth-$TIMESTAMP --quiet + + # Update manifest file for linux-amd64 builds only + aws s3 cp s3://iliad-geth-binaries/geth/manifest.txt manifest.txt --quiet || touch manifest.txt + echo "$TIMESTAMP" >> manifest.txt + aws s3 cp manifest.txt s3://iliad-geth-binaries/geth/manifest.txt --quiet + fi + + mkdir $FOLDER_NAME + mv $binary_name $FOLDER_NAME/ + + echo "Archiving the geth binary..." + tar -czvf $ARCHIVE_NAME $FOLDER_NAME + + if [ $? -ne 0 ]; then + echo "Failed to create the archive: $ARCHIVE_NAME" + exit 1 + fi + + echo "Uploading $ARCHIVE_NAME to S3..." + aws s3 cp $ARCHIVE_NAME s3://iliad-geth-binaries/geth-public/$ARCHIVE_NAME --quiet + + if [ $? -ne 0 ]; then + echo "Failed to upload $ARCHIVE_NAME to S3!" + exit 1 + fi + + cleanup: + runs-on: ubuntu-latest + needs: build_and_push + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::478656756051:role/iac-max-role + aws-region: us-west-1 + role-session-name: github-actions + - name: Cleanup old binaries run: | - # List objects in the bucket and sort by LastModified date - aws s3api list-objects-v2 --bucket iliad-geth-binaries --prefix geth/ --query "sort_by(Contents,&LastModified)[*].Key" > all_binaries.json - - # Extract the list of keys, remove the latest NUM_BINARIES_TO_KEEP - BINARIES_TO_DELETE=$(jq -r ".[0:-${NUM_BINARIES_TO_KEEP}][]" all_binaries.json) - - if [ -n "$BINARIES_TO_DELETE" ]; then - # Delete old binaries - for key in $BINARIES_TO_DELETE; do - aws s3 rm s3://iliad-geth-binaries/$key --quiet - done - echo "Deleted old binaries: $BINARIES_TO_DELETE" - else - echo "No old binaries to delete." - fi + cleanup_s3() { + PREFIX=$1 + KEEP=$2 + + echo "Cleaning up in bucket iliad-geth-binaries with prefix: $PREFIX, keeping latest $KEEP binaries" + + aws s3api list-objects-v2 --bucket iliad-geth-binaries --prefix $PREFIX --query "sort_by(Contents,&LastModified)[*].Key" > all_binaries.json + + # Extract the list of keys, remove the latest $KEEP binaries + BINARIES_TO_DELETE=$(jq -r ".[0:-${KEEP}][]" all_binaries.json) + + if [ -n "$BINARIES_TO_DELETE" ]; then + # Delete old binaries + for key in $BINARIES_TO_DELETE; do + aws s3 rm s3://iliad-geth-binaries/$key --quiet + done + echo "Deleted old binaries: $BINARIES_TO_DELETE" + else + echo "No old binaries to delete." + fi + } + + # Cleanup internal geth binaries + cleanup_s3 "geth/" "${NUM_INTERNAL_BINARIES_TO_KEEP}" + + # Cleanup public geth binaries + cleanup_s3 "geth-public/" "${NUM_PUBLIC_BINARIES_TO_KEEP}"