From 7181f4c71202c66948234b2d199cb3d4043e818a Mon Sep 17 00:00:00 2001 From: lystopad Date: Fri, 6 Dec 2024 12:17:00 +0100 Subject: [PATCH] Add support for building Debian packages (arm/amd) in the release workflow. (#13024) See https://github.com/erigontech/erigon/issues/12891 for more details. Changes: - new reusable workflow for building debian packages - this reusable workflow is a part of release workflow. Binaries which is built during build stage used to build docker image, debian package and to publish as release artifacts. --- .github/workflows/release.yml | 266 +++++++++++++----- .../reusable-release-build-debian-pkg.yml | 139 +++++++++ 2 files changed, 333 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/reusable-release-build-debian-pkg.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56d9d3de3cc..49a8c866407 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Release +run-name: "Build release ${{ inputs.release_version}} from branch ${{ inputs.checkout_ref }} by @${{ github.actor}}" env: APPLICATION: "erigon" @@ -7,7 +8,7 @@ env: APP_REPO: "erigontech/erigon" PACKAGE: "github.com/erigontech/erigon" DOCKERHUB_REPOSITORY: "erigontech/erigon" - DOCKERFILE_PATH: "./Dockerfile.release" + DOCKERFILE_PATH: "Dockerfile.release" GITHUB_AUTOMATION_EMAIL: "github-automation@erigon.tech" GITHUB_AUTOMATION_NAME: "Erigon Github Automation" LABEL_DESCRIPTION: "Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default." @@ -16,12 +17,6 @@ on: push: branches-ignore: - '**' - #branches: - # - 'master' - #tags: - ## only trigger on release tags: - #- 'v*.*.*' - #- 'v*.*.*-*' workflow_dispatch: inputs: checkout_ref: @@ -50,8 +45,13 @@ jobs: build-release: ## runs-on: ubuntu-22.04 runs-on: ubuntu-latest-devops-xxlarge - timeout-minutes: 60 - name: Build Artifacts and multi-platform Docker image, publish draft of the Release Notes + timeout-minutes: 75 + name: Create git tag, build and publish Artifacts + outputs: + commit-id: ${{ steps.getCommitId.outputs.id }} + short-commit-id: ${{ steps.getCommitId.outputs.short_commit_id }} + application: ${{ env.APPLICATION }} + parsed-version: ${{ steps.getCommitId.outputs.parsed_version}} steps: - name: Checkout git repository ${{ env.APP_REPO }} @@ -69,6 +69,9 @@ jobs: exit 1 else echo "OK: tag ${{ inputs.release_version }} does not exists. Proceeding." + git tag ${{ inputs.release_version }} + git push origin ${{ inputs.release_version }} + echo; echo "Git TAG ${{ inputs.release_version }} created and pushed." fi - name: Get commit id @@ -76,12 +79,7 @@ jobs: run: | echo "id=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT echo "short_commit_id=$(git rev-parse --short=7 HEAD)" >> $GITHUB_OUTPUT - - - name: Login to Docker Hub - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 ## v3.3.0 - with: - username: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_USERNAME }} - password: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }} + echo "parsed_version=$(echo ${{ inputs.release_version }} | sed -e 's/^v//g')" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf ## v3.2.0 @@ -110,37 +108,6 @@ jobs: echo "DEBUG: content of the dist/ directory" find dist/ -ls - - name: Build and push multi-platform docker images (${{ env.BUILD_VERSION }} and latest) in case perform_release is true - if: ${{ inputs.perform_release }} - env: - BUILD_VERSION: ${{ inputs.release_version }} - DOCKER_URL: ${{ env.DOCKERHUB_REPOSITORY }} - DOCKER_PUBLISH_LATEST_CONDITION: ${{ inputs.publish_latest_tag && format('--tag {0}:latest ',env.DOCKERHUB_REPOSITORY) || '' }} - run: | - docker buildx build \ - --file ${{ env.DOCKERFILE_PATH }} \ - --build-arg RELEASE_DOCKER_BASE_IMAGE=${{ env.DOCKER_BASE_IMAGE }} \ - --build-arg VERSION=${{ env.BUILD_VERSION }} \ - --build-arg APPLICATION=${{ env.APPLICATION }} \ - --tag ${{ env.DOCKER_URL }}:${{ env.BUILD_VERSION }} \ - --target release \ - --attest type=provenance,mode=max \ - --sbom=true \ - ${{ env.DOCKER_PUBLISH_LATEST_CONDITION }} \ - --label org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --label org.opencontainers.image.authors="https://github.com/erigontech/erigon/graphs/contributors" \ - --label org.opencontainers.image.url="https://github.com/erigontech/erigon/blob/main/Dockerfile" \ - --label org.opencontainers.image.documentation="https://github.com/erigontech/erigon/blob/main/Dockerfile" \ - --label org.opencontainers.image.source="https://github.com/erigontech/erigon/blob/main/Dockerfile" \ - --label org.opencontainers.image.version=${{ inputs.release_version }} \ - --label org.opencontainers.image.revision=${{ steps.getCommitId.outputs.id }} \ - --label org.opencontainers.image.vcs-ref-short=${{ steps.getCommitId.outputs.short_commit_id }} \ - --label org.opencontainers.image.vendor="${{ github.repository_owner }}" \ - --label org.opencontainers.image.description="${{ env.LABEL_DESCRIPTION }}" \ - --label org.opencontainers.image.base.name="${{ env.DOCKER_BASE_IMAGE }}" \ - --push \ - --platform linux/amd64,linux/amd64/v2,linux/arm64 . - - name: Upload artifact -- linux/arm64 uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a ## v4.3.6 with: @@ -186,30 +153,185 @@ jobs: compression-level: 0 if-no-files-found: error -## not required for now -- commented: -# - name: Create and push a git tag for the released version in case perform_release is set -# if: ${{ inputs.perform_release }} -# run: | -# git config --global user.email ${{ env.GITHUB_AUTOMATION_EMAIL }} -# git config --global user.name "${{ env.GITHUB_AUTOMATION_NAME }}" -# git tag -a ${{ inputs.release_version }} -m "Release ${{ inputs.release_version }}" -# git push origin ${{ inputs.release_version }} - - - name: Publish draft of the Release notes with assets in case perform_release is set - if: ${{ inputs.perform_release }} - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - DOCKER_TAGS: ${{ env.DOCKERHUB_REPOSITORY }}:${{ inputs.release_version }} - GITHUB_RELEASE_TARGET: ${{ inputs.checkout_ref }} + + + build-debian-pkg: + name: Debian packages + needs: [ build-release ] + uses: erigontech/erigon/.github/workflows/reusable-release-build-debian-pkg.yml@main + with: + application: ${{ needs.build-release.outputs.application }} + version: ${{ needs.build-release.outputs.parsed-version }} + + + + publish-docker-image: + needs: [ build-release ] + runs-on: ubuntu-latest + timeout-minutes: 30 + name: Docker image + + steps: + + - name: Fast checkout just ${{ env.DOCKERFILE_PATH }} from git repository ${{ env.APP_REPO }} + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 ## 4.1.7 release + with: + repository: ${{ env.APP_REPO }} + sparse-checkout: ${{ env.DOCKERFILE_PATH }} + sparse-checkout-cone-mode: false + ref: ${{ needs.build-release.outputs.commit-id }} + + - name: Download arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_arm64.tar.gz + path: dist/ + + - name: Download amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64.tar.gz + path: dist/ + + - name: Download amd64v2 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64v2.tar.gz + path: dist/ + + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf ## v3.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db ## v3.6.1 + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 ## v3.3.0 + with: + username: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_USERNAME }} + password: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }} + + - name: Build and push multi-platform docker images (${{ env.BUILD_VERSION }} and maybe latest) in case perform_release is true + if: ${{ inputs.perform_release }} + env: + BUILD_VERSION: ${{ inputs.release_version }} + DOCKER_URL: ${{ env.DOCKERHUB_REPOSITORY }} + DOCKER_PUBLISH_LATEST_CONDITION: ${{ inputs.publish_latest_tag && format('--tag {0}:latest ',env.DOCKERHUB_REPOSITORY) || '' }} + run: | + pwd + find . -ls + docker buildx build \ + --file ${{ env.DOCKERFILE_PATH }} \ + --build-arg RELEASE_DOCKER_BASE_IMAGE=${{ env.DOCKER_BASE_IMAGE }} \ + --build-arg VERSION=${{ env.BUILD_VERSION }} \ + --build-arg APPLICATION=${{ env.APPLICATION }} \ + --tag ${{ env.DOCKER_URL }}:${{ env.BUILD_VERSION }} \ + --target release \ + --attest type=provenance,mode=max \ + --sbom=true \ + ${{ env.DOCKER_PUBLISH_LATEST_CONDITION }} \ + --label org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --label org.opencontainers.image.authors="https://github.com/erigontech/erigon/graphs/contributors" \ + --label org.opencontainers.image.url="https://github.com/erigontech/erigon/blob/${{ inputs.checkout_ref }}/${{ env.DOCKERFILE_PATH }}" \ + --label org.opencontainers.image.documentation="https://github.com/erigontech/erigon/blob/${{ inputs.checkout_ref }}/${{ env.DOCKERFILE_PATH }}" \ + --label org.opencontainers.image.source="https://github.com/erigontech/erigon/blob/${{ inputs.checkout_ref }}/${{ env.DOCKERFILE_PATH }}" \ + --label org.opencontainers.image.version=${{ inputs.release_version }} \ + --label org.opencontainers.image.revision=${{ needs.build-release.outputs.commit-id }} \ + --label org.opencontainers.image.vcs-ref-short=${{ needs.build-release.outputs.short-commit-id }} \ + --label org.opencontainers.image.vendor="${{ github.repository_owner }}" \ + --label org.opencontainers.image.description="${{ env.LABEL_DESCRIPTION }}" \ + --label org.opencontainers.image.base.name="${{ env.DOCKER_BASE_IMAGE }}" \ + --push \ + --platform linux/amd64,linux/amd64/v2,linux/arm64 . + + + + publish-release: + needs: [ build-debian-pkg, publish-docker-image, build-release ] + runs-on: ubuntu-latest + timeout-minutes: 15 + name: Publish release notes + + steps: + - name: Download linux/arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_arm64.tar.gz + path: dist/ + + - name: Download linux/amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64.tar.gz + path: dist/ + + - name: Download linux/amd64v2 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64v2.tar.gz + path: dist/ + + - name: Download darwin/amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_amd64.tar.gz + path: dist/ + + - name: Download darwin/arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_arm64.tar.gz + path: dist/ + + - name: Download arm64 debian package + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ needs.build-release.outputs.parsed-version }}_arm64.deb + path: dist/ + + - name: Download amd64 debian package + uses: actions/download-artifact@v4 + with: + name: ${{ env.APPLICATION }}_${{ needs.build-release.outputs.parsed-version }}_amd64.deb + path: dist/ + + - name: Publish draft of the Release notes with assets in case perform_release is set + if: ${{ inputs.perform_release }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + DOCKER_TAGS: ${{ env.DOCKERHUB_REPOSITORY }}:${{ inputs.release_version }} + GITHUB_RELEASE_TARGET: ${{ inputs.checkout_ref }} + run: | + cd dist + sha256sum *.tar.gz *.deb > ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt + gh release create \ + --generate-notes \ + --target ${GITHUB_RELEASE_TARGET} \ + --draft=true \ + --title "${{ inputs.release_version }}" \ + --notes "**Improvements:**
- ...coming soon

**Bugfixes:**

- ...coming soon

**Docker images:**

Docker image released:
${{ env.DOCKER_TAGS }}

... coming soon
" \ + "${{ inputs.release_version }}" \ + *.tar.gz *.deb ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt + + + In-case-of-failure: + name: "In case of failure: remove remote git tag pointing to the new version." + needs: [ publish-release, build-release ] + if: always() && !contains(needs.build-release.result, 'success') + runs-on: ubuntu-22.04 + + steps: + - name: Checkout git repository ${{ env.APP_REPO }} reference ${{ inputs.checkout_ref }} + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 ## 4.1.7 release + with: + repository: ${{ env.APP_REPO }} + fetch-depth: 0 + ref: ${{ inputs.checkout_ref }} + path: 'erigon' + + - name: Rollback - remove git tag ${{ inputs.release_version }} + if: ${{ (inputs.perform_release) && (inputs.release_version != '') }} run: | - cd dist - sha256sum *.tar.gz > ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt - gh release create \ - --generate-notes \ - --target ${GITHUB_RELEASE_TARGET} \ - --draft=true \ - --title "${{ inputs.release_version }}" \ - --notes "**Improvements:**
- ...coming soon

**Bugfixes:**

- ...coming soon

**Docker images:**

Docker image released:
${{ env.DOCKER_TAGS }}

... coming soon
" \ - "${{ inputs.release_version }}" \ - *.tar.gz ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt \ No newline at end of file + cd erigon + git push -d origin ${{ inputs.release_version }} diff --git a/.github/workflows/reusable-release-build-debian-pkg.yml b/.github/workflows/reusable-release-build-debian-pkg.yml new file mode 100644 index 00000000000..4ebf47769e9 --- /dev/null +++ b/.github/workflows/reusable-release-build-debian-pkg.yml @@ -0,0 +1,139 @@ +name: Build debian package (part of release process) + +on: + workflow_call: + inputs: + application: + required: true + type: string + version: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Download arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.application }}_v${{ inputs.version }}_linux_arm64.tar.gz + + - name: Download amd64v2 artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.application }}_v${{ inputs.version }}_linux_amd64v2.tar.gz + + - name: Update and install required packages, run dpkg + run: | + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -y dpkg-dev debhelper + sudo dpkg --clear-avail + + - name: Extract archives and rename amd64v2 to amd64 + run: | + tar xzvf ${{ inputs.application }}_v${{ inputs.version }}_linux_amd64v2.tar.gz + mv -v ${{ inputs.application }}_v${{ inputs.version }}_linux_amd64v2 ${{ inputs.application }}_v${{ inputs.version }}_linux_amd64 + tar xzvf ${{ inputs.application }}_v${{ inputs.version }}_linux_arm64.tar.gz + cat <<-END > postinst.template + #!/bin/bash + echo "WARNING: erigon package does not install any configurations nor services." + echo "Use your specific way to configure and run erigon according to your needs." + echo "More details on how to run erigon could be found at https://erigon.gitbook.io/erigon ." + END + + # Creating directory structure + # see https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + # https://www.debian.org/doc/manuals/developers-reference/best-pkging-practices.html + - name: Build debian package for amd64 + env: + ARCH: "amd64" + run: | + mkdir -p deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/usr/bin \ + deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN + install postinst.template deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN/postinst + cat <<-END > deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN/control + Package: ${{ inputs.application }} + Version: ${{ inputs.version }} + Section: misc + Priority: optional + Architecture: ${ARCH} + Maintainer: Erigon DevOps [https://github.com/erigontech/erigon/issues] + Description: Erigon - Ethereum implementation on the efficiency frontier + Vcs-Git: https://github.com/erigontech/erigon.git + Vcs-Browser: https://github.com/erigontech/erigon + END + install -v -p ${{ inputs.application }}_v${{ inputs.version }}_linux_${ARCH}/* \ + deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/usr/bin + dpkg-deb --build --root-owner-group deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH} + + - name: Build debian package for arm64 + env: + ARCH: "arm64" + run: | + mkdir -p deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/usr/bin \ + deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN + install postinst.template deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN/postinst + cat <<-END > deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN/control + Package: ${{ inputs.application }} + Version: ${{ inputs.version }} + Section: misc + Priority: optional + Architecture: ${ARCH} + Maintainer: Erigon DevOps [https://github.com/erigontech/erigon/issues] + Description: Erigon - Ethereum implementation on the efficiency frontier + Vcs-Git: https://github.com/erigontech/erigon.git + Vcs-Browser: https://github.com/erigontech/erigon + END + echo "debug start" + cat deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/DEBIAN/control + echo "debug end" + install -v -p ${{ inputs.application }}_v${{ inputs.version }}_linux_${ARCH}/* \ + deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH}/usr/bin + dpkg-deb --build --root-owner-group deb-pkg/${{ inputs.application }}_${{ inputs.version }}_${ARCH} + + - name: Debug output + run: | + cd ./deb-pkg + sha256sum ${{ inputs.application }}_${{ inputs.version }}_amd64.deb > ${{ inputs.application }}_${{ inputs.version }}_amd64.deb.checksum + sha256sum ${{ inputs.application }}_${{ inputs.version }}_arm64.deb > ${{ inputs.application }}_${{ inputs.version }}_arm64.deb.checksum + ls -l *deb *.checksum + + - name: Apload artifact amd64.deb + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a ## v4.3.6 + with: + name: ${{ inputs.application }}_${{ inputs.version }}_amd64.deb + path: ./deb-pkg/${{ inputs.application }}_${{ inputs.version }}_amd64.deb + retention-days: 5 + compression-level: 0 + if-no-files-found: error + + - name: Apload artifact amd64.deb.checksum + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a ## v4.3.6 + with: + name: ${{ inputs.application }}_${{ inputs.version }}_amd64.deb.checksum + path: ./deb-pkg/${{ inputs.application }}_${{ inputs.version }}_amd64.deb.checksum + retention-days: 5 + compression-level: 0 + if-no-files-found: error + + - name: Apload artifact arm64.deb + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a ## v4.3.6 + with: + name: ${{ inputs.application }}_${{ inputs.version }}_arm64.deb + path: ./deb-pkg/${{ inputs.application }}_${{ inputs.version }}_arm64.deb + retention-days: 5 + compression-level: 0 + if-no-files-found: error + + - name: Apload artifact arm64.deb.checksum + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a ## v4.3.6 + with: + name: ${{ inputs.application }}_${{ inputs.version }}_arm64.deb.checksum + path: ./deb-pkg/${{ inputs.application }}_${{ inputs.version }}_arm64.deb.checksum + retention-days: 5 + compression-level: 0 + if-no-files-found: error \ No newline at end of file