[Release|CI/CD] Github pipeline to publish polkadot deb package (#6640)

This pipeline should replace a manual action done on the `cleamroom` server to publish the `polkadot` deb package to our apt repo with the pipeline triggered from the new paritytech-release org. Right now, this is done manually by running the [add-packages.sh](https://github.com/paritytech/cleanroom/blob/master/ansible/roles/parity-repos/files/add-packages.sh) script on the `cleanroom` machine. What is done under the hood: - Pipeline downloads `polakdot` deb package from S3, that was prebuilt in the [Build release rc pipeline](https://github.com/paritytech/polkadot-sdk/blob/master/.github/workflows/release-build-rc.yml) - Prepares and syncs local apt repository - Adds and signs deb package to it using `reprepro` - Uploads new deb package to the distributed repo Closes: paritytech/release-engineering#239
paritytech · Nov 26, 2024 · dbb2fd9 · dbb2fd9
1 parent 9ab0616
commit dbb2fd9
Show file tree

Hide file tree

Showing 3 changed files with 222 additions and 2 deletions.
diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh
@@ -237,15 +237,44 @@ fetch_release_artifacts() {
   popd > /dev/null
 }
 
-# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# Fetch deb package from S3. Assumes the ENV are set:
 # - RELEASE_ID
 # - GITHUB_TOKEN
 # - REPO in the form paritytech/polkadot
-fetch_release_artifacts_from_s3() {
+fetch_debian_package_from_s3() {
   BINARY=$1
   echo "Version    : $VERSION"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
+  echo "Tag        : $RELEASE_TAG"
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
+
+  URL_BASE=$(get_s3_url_base $BINARY)
+  echo "URL_BASE=$URL_BASE"
+
+  URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
+
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
+
+  echo "Fetching deb package..."
+
+  echo "Fetching %s" "$URL"
+  curl --progress-bar -LO "$URL" || echo "Missing $URL"
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+
+}
+
+# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts_from_s3() {
+  BINARY=$1
   OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 

diff --git a/.github/scripts/release/distributions b/.github/scripts/release/distributions
@@ -0,0 +1,39 @@
+Origin: Parity
+Label: Parity
+Codename: release
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity Staging
+Codename: staging
+Architectures: amd64
+Components: main
+Description: Staging distribution for Parity Technologies Ltd. packages
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2407
+Codename: stable2407
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2409
+Codename: stable2409
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2412
+Codename: stable2412
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
diff --git a/.github/workflows/release-40_publish-deb-package.yml b/.github/workflows/release-40_publish-deb-package.yml
@@ -0,0 +1,152 @@
+name: Release - Publish polakdot deb package
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Current final release tag in the format polakdot-stableYYMM or polkadot-stable-YYMM-X
+        default: polkadot-stable2412
+        required: true
+        type: string
+
+      distribution:
+        description: Distribution where to publish deb package (release, staging, stable2407, etc)
+        default: staging
+        required: true
+        type: string
+
+jobs:
+  check-synchronization:
+    uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main
+
+  validate-inputs:
+    needs: [check-synchronization]
+    if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+        release_tag: ${{ steps.validate_inputs.outputs.release_tag }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Validate inputs
+        id: validate_inputs
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          RELEASE_TAG=$(validate_stable_tag ${{ inputs.tag }})
+          echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
+
+
+  fetch-artifacts-from-s3:
+    runs-on: ubuntu-latest
+    needs: [validate-inputs]
+    env:
+      REPO: ${{ github.repository }}
+      RELEASE_TAG: ${{ needs.validate-inputs.outputs.release_tag }}
+    outputs:
+      VERSION: ${{ steps.fetch_artifacts_from_s3.outputs.VERSION }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Fetch rc artifacts or release artifacts from s3 based on version
+        id: fetch_artifacts_from_s3
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          VERSION="$(get_polkadot_node_version_from_code)"
+          echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
+
+          fetch_debian_package_from_s3 polkadot
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: release-artifacts
+          path: release-artifacts/polkadot/*.deb
+
+  publish-deb-package:
+    runs-on: ubuntu-latest
+    needs: [fetch-artifacts-from-s3]
+    environment: release
+    env:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_DEB_PATH: "s3://releases-package-repos/deb"
+      LOCAL_DEB_REPO_PATH: ${{ github.workspace }}/deb
+      VERSION: ${{ needs.fetch-artifacts-from-s3.outputs.VERSION }}
+
+    steps:
+      - name: Install pgpkkms
+        run: |
+          # Install pgpkms that is used to sign built artifacts
+          python3 -m pip install "pgpkms @ git+https://github.com/paritytech-release/pgpkms.git@1f8555426662ac93a3849480a35449f683b1c89f"
+          echo "PGPKMS_REPREPRO_PATH=$(which pgpkms-reprepro)" >> $GITHUB_ENV
+
+      - name: Install awscli
+        run: |
+          python3 -m pip install awscli
+          which aws
+
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Import gpg keys
+        shell: bash
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          import_gpg_keys
+
+      - name: Download artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: release-artifacts
+          path: release-artifacts
+
+      - name: Setup local deb repo
+        run: |
+          sudo apt-get install -y reprepro
+          which reprepro
+
+          sed -i "s|^SignWith:.*|SignWith: ! ${PGPKMS_REPREPRO_PATH}|" ${{ github.workspace }}/.github/scripts/release/distributions
+
+          mkdir -p ${{ github.workspace }}/deb/conf
+          cp ${{ github.workspace }}/.github/scripts/release/distributions ${{ github.workspace }}/deb/conf/distributions
+          cat ${{ github.workspace }}/deb/conf/distributions
+
+      - name: Sync local deb repo
+        env:
+          AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+        run: |
+          # Download the current state of the deb repo
+          aws s3 sync "$AWS_DEB_PATH/db" "$LOCAL_DEB_REPO_PATH/db"
+          aws s3 sync "$AWS_DEB_PATH/pool" "$LOCAL_DEB_REPO_PATH/pool"
+          aws s3 sync "$AWS_DEB_PATH/dists" "$LOCAL_DEB_REPO_PATH/dists"
+
+      - name: Add deb package to local repo
+        env:
+          PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+          PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+          AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          # Add the new deb to the repo
+          reprepro -b "$LOCAL_DEB_REPO_PATH" includedeb "${{ inputs.distribution }}" "release-artifacts/polkadot_${VERSION}_amd64.deb"
+
+      - name: Upload updated deb repo
+        env:
+          AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+        run: |
+          # Upload the updated repo - dists and pool should be publicly readable
+          aws s3 sync "$LOCAL_DEB_REPO_PATH/pool" "$AWS_DEB_PATH/pool" --acl public-read
+          aws s3 sync "$LOCAL_DEB_REPO_PATH/dists" "$AWS_DEB_PATH/dists" --acl public-read
+          aws s3 sync "$LOCAL_DEB_REPO_PATH/db" "$AWS_DEB_PATH/db"
+          aws s3 sync "$LOCAL_DEB_REPO_PATH/conf" "$AWS_DEB_PATH/conf"
+
+          # Invalidate caches to make sure latest files are served
+          aws cloudfront create-invalidation --distribution-id E36FKEYWDXAZYJ --paths '/deb/*'