Ci/ubuntu migration (#2268)

* change builder image to ubuntu * change versions in Makefile * zombie tests and runner -> ubuntu * rename kagome_builder_deb -> kagome_builder * push builder to dockerhub * build tidy from dockerhub * zombie builder custom tag * polkadot builder image -> ubuntu * add rust version to zombie tester * zombie tester version info in log fix * update custom build tags * WASMEDGE_ID update * kagome runner image fix * zombie-tester custom tag * custom zombie tester for runtime cache build * zombie tests action update * update runtime package version format --------- Co-authored-by: Kirill Azovtsev <[email protected]>
qdrvm · Nov 12, 2024 · 18889a9 · 18889a9
1 parent 378ba81
commit 18889a9
Show file tree

Hide file tree

Showing 15 changed files with 337 additions and 130 deletions.
diff --git a/.github/workflows/kagome-builder-deb.yaml b/.github/workflows/kagome-builder-deb.yaml
@@ -4,29 +4,41 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-name: Kagome Builder | Debian (Bookworm)
+name: Kagome Builder | Ubuntu (24.04_LTS)
 
 on:
   #  schedule:
   #    - cron: '0 2 * * 0' # Every Sunday at 2:00 AM
   workflow_dispatch:
     inputs:
       kagome_image_rebuild:
-        description: 'Rebuild Kagome Builder Minideb image'
+        description: 'Rebuild Kagome Builder image'
         required: false
         default: 'false'
         type: 'boolean'
       runtime_cache_rebuild:
-        description: 'Rebuild Kagome Runtime Cache'
+        description: 'Rebuild Kagome Runtime cache'
         required: false
         default: 'false'
         type: 'boolean'
+      builder_latest_tag:
+        description: 'Custom Kagome Builder tag'
+        default: 'latest'
+        required: false
+        type: 'string'
+      zombie_tester_latest_tag:
+        description: 'Custom Zombie Tester tag'
+        default: 'latest'
+        required: false
+        type: 'string'
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
+  BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }}
+  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
 
 jobs:
-  build_polkadot_builder:
+  build_kagome_builder:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -50,28 +62,42 @@ jobs:
         id: check-tag
         working-directory: ./housekeeping/docker/kagome-dev
         run: |
-          IMAGE_TAG=$(grep '^MINIDEB_IMAGE' Makefile | awk -F '[@:]' '{print $3}' | cut -c 1-7)_rust-$(grep '^RUST_VERSION' Makefile | sed 's/.*?= //')
-          KAGOME_BUILDER_DEB_IMAGE=${DOCKER_REGISTRY_PATH}kagome_builder_deb:${IMAGE_TAG}
-          TAG_EXISTS=$(docker manifest inspect ${KAGOME_BUILDER_DEB_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false")
+          IMAGE_TAG=$(make builder_image_tag)
+          KAGOME_BUILDER_IMAGE=${DOCKER_REGISTRY_PATH}kagome_builder_deb:${IMAGE_TAG}
+          TAG_EXISTS=$(docker manifest inspect ${KAGOME_BUILDER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false")
           echo "exists=$TAG_EXISTS" >> $GITHUB_ENV
-          echo "kagome_builder_deb=$KAGOME_BUILDER_DEB_IMAGE" >> $GITHUB_OUTPUT
+          echo "kagome_builder=$KAGOME_BUILDER_IMAGE" >> $GITHUB_OUTPUT
+
+      - name: "Build Kagome Builder"
+        working-directory: ./housekeeping/docker/kagome-dev
+        if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true'
+        run: make kagome_builder \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          BUILDER_LATEST_TAG=${BUILDER_LATEST_TAG}
 
-      - name: "Build Polkadot Builder"
+      - name: "Push Kagome Builder"
         working-directory: ./housekeeping/docker/kagome-dev
         if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true'
-        run: make kagome_builder_deb \
+        run: make kagome_builder_push \
           DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH}
 
-      - name: "Push Polkadot Builder"
+      - name: "Login to Docker Hub"
+        uses: docker/login-action@v3
+        if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: "Push Kagome Builder (DockerHub)"
         working-directory: ./housekeeping/docker/kagome-dev
         if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true'
-        run: make kagome_builder_deb_push \
+        run: make kagome_builder_push_dockerhub \
           DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH}
 
       - name: "Versions Info"
         if: always()
         run: |
-          echo "Docker Image Name: ${{ steps.check-tag.outputs.kagome_builder_deb }}"
+          echo "Docker Image Name: ${{ steps.check-tag.outputs.kagome_builder }}"
 
   build_kagome_runtime:
     runs-on: [ actions-runner-controller ]
@@ -97,7 +123,8 @@ jobs:
         working-directory: ./housekeeping/docker/kagome-dev
         if: github.event.inputs.runtime_cache_rebuild == 'true'
         run: make runtime_cache \
-          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH}
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          TESTER_LATEST_TAG=${{ env.TESTER_LATEST_TAG }}
 
       - name: "Push Runtime Cache"
         working-directory: ./housekeeping/docker/kagome-dev

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -29,6 +29,16 @@ on:
         options:
           - 'true'
           - 'false'
+      builder_latest_tag:
+        description: 'Custom Builder tag'
+        default: 'latest'
+        required: false
+        type: 'string'
+      zombie_tester_latest_tag:
+        description: 'Custom Zombie Tester tag'
+        default: 'latest'
+        required: false
+        type: 'string'
 
 env:
   BUILD_DIR: build
@@ -52,6 +62,8 @@ env:
   DEFAULT_BUILD_TYPE: Release
   GIT_REF_NAME: ${{ github.ref_name }}
   IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }}
+  BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }}
+  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
 
 
 jobs:
@@ -205,19 +217,6 @@ jobs:
         if: github.ref != 'refs/heads/master'
         run: git fetch origin master:master || true
 
-      - name: "Authenticate with Google Cloud"
-        uses: 'google-github-actions/auth@v2'
-        with:
-          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-
-      - name: "Set up Cloud SDK"
-        uses: 'google-github-actions/setup-gcloud@v2'
-
-      - name: "Configure Docker for GCR"
-        run: |
-          gcloud auth configure-docker --quiet
-          gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
-
       - name: "Get commit version"
         working-directory: ./housekeeping/docker/kagome-dev
         run: make get_versions
@@ -242,9 +241,9 @@ jobs:
         working-directory: ./housekeeping/docker/kagome-dev
         run:
           make kagome_dev_docker_build_tidy \
-          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
           GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
           GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
+          BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
           CI="true"
 
       - name: "Cleaning cache"
@@ -271,11 +270,11 @@ jobs:
       fail-fast: false
       matrix:
         options:
-          - name: "Self-hosted: Minideb: Build Debug"
+          - name: "Self-hosted docker: Build Debug"
             build_type: "Debug"
-          - name: "Self-hosted: Minideb: Build Release"
+          - name: "Self-hosted docker: Build Release"
             build_type: "Release"
-          - name: "Self-hosted: Minideb: Build RelWithDebInfo"
+          - name: "Self-hosted docker: Build RelWithDebInfo"
             build_type: "RelWithDebInfo"
     name: "${{ matrix.options.name }}"
 
@@ -338,6 +337,7 @@ jobs:
           DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
           GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
           GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
+          BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
           BUILD_TYPE=${{ matrix.options.build_type }}
 
       - name: "Push Kagome APT Package"
@@ -348,7 +348,7 @@ jobs:
           make upload_apt_package \
           BUILD_TYPE=${{ matrix.options.build_type }}
 
-      - name: Set up Docker Buildx
+      - name: "Set up Docker Buildx"
         if: env.package_exist == 'False'
         uses: docker/[email protected]
 
@@ -500,7 +500,7 @@ jobs:
 
       - name: "Get Zombie Tester Version"
         run: |
-          ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:latest
+          ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${{ env.TESTER_LATEST_TAG }}
           echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}"
           echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV
 

diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml
@@ -30,6 +30,16 @@ on:
         description: 'Custom Polkadot SDK tag'
         required: false
         type: 'string'
+      builder_latest_tag:
+        description: 'Custom Polkadot Builder tag'
+        default: 'latest'
+        required: false
+        type: 'string'
+      zombie_tester_latest_tag:
+        description: 'Custom Zombie Tester tag'
+        default: 'latest'
+        required: false
+        type: 'string'
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
@@ -38,6 +48,8 @@ env:
   CACHE_VERSION: v001
   CACHE_PATHS: ./zombienet/docker/cargo
   POLKADOT_SDK_TAG: ${{ github.event.inputs.polkadot_sdk_tag }}
+  BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }}
+  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
 
 jobs:
   build_polkadot_builder:
@@ -79,6 +91,7 @@ jobs:
         if: env.exists == 'false' || github.event.inputs.polkadot_image_rebuild == 'true'
         run: make polkadot_builder \
           DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH}
+          BUILDER_LATEST_TAG=${BUILDER_LATEST_TAG}
 
       - name: "Push Polkadot Builder"
         working-directory: ./zombienet/docker
@@ -214,16 +227,17 @@ jobs:
           echo "exists=$TAG_EXISTS" >> $GITHUB_ENV
           echo "zombie_tester_image=$ZOMBIE_TESTER_IMAGE" >> $GITHUB_OUTPUT
 
-      - name: "Build Polkadot Builder"
+      - name: "Build Zombie Tester"
         working-directory: ./zombienet/docker
         if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true'
         run: make zombie_tester \
           DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
           GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} \
           PROJECT_ID=${{ secrets.GCP_PROJECT }} \
+          TESTER_LATEST_TAG=${{ env.TESTER_LATEST_TAG }} \
           POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_BINARY_PACKAGE_VERSION}"
 
-      - name: "Push Polkadot Builder"
+      - name: "Push Zombie Tester"
         working-directory: ./zombienet/docker
         if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true'
         run: make zombie_tester_push \

diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml
@@ -34,6 +34,16 @@ on:
         options:
           - 'true'
           - 'false'
+      builder_latest_tag:
+        description: 'Custom Builder tag'
+        default: 'latest'
+        required: false
+        type: 'string'
+      zombie_tester_latest_tag:
+        description: 'Custom Zombie Tester tag'
+        default: 'latest'
+        required: false
+        type: 'string'
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
@@ -45,6 +55,8 @@ env:
   CACHE_VERSION: v01
   CACHE_PATH: ./build/cache
   USE_CACHE: ${{ github.event.inputs.use_cache || 'true' }}
+  BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }}
+  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
 
 jobs:
   kagome_dev_docker_build:
@@ -111,6 +123,7 @@ jobs:
           GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
           GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
           BUILD_TYPE=${{ github.event.inputs.build_type }} \
+          BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
           WERROR=${{ github.event.inputs.werror }}
 
       - name: "Push Kagome APT Package"
@@ -236,7 +249,7 @@ jobs:
           
       - name: "Get Zombie Tester Version"
         run: |
-          ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:latest
+          ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${{ env.TESTER_LATEST_TAG }}
           echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}"
           echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV
         

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -38,7 +38,7 @@ set(CMAKE_TOOLCHAIN_FILE
 # SHA1 hash of the WasmEdge repository archive.
 # Required to separate cached runtimes
 # compiled with different WasmEdge versions.
-set(WASMEDGE_ID 58aea400de9179ad3e314c7e84fd4da345b8a643)
+set(WASMEDGE_ID 00c9a478de6d1d9443d7b6df5a5a488f9c84b716)
 
 include("cmake/Hunter/init.cmake")
 

diff --git a/docs/source/overview/getting_started.md b/docs/source/overview/getting_started.md
@@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0
 
 ### Prerequisites
 
-For now, please refer to the [Dockerfile](https://github.com/soramitsu/kagome/blob/master/housekeeping/docker/kagome-dev/minideb.Dockerfile) to get a picture of what you need for a local build-environment.
+For now, please refer to the [Dockerfile](https://github.com/soramitsu/kagome/blob/master/housekeeping/docker/kagome-dev/kagome_runner.Dockerfile) to get a picture of what you need for a local build-environment.
 
 ### Clone