From 182395ca922475b2faf9f50fda039abdb521e1ed Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 1 Feb 2024 17:26:11 +0300 Subject: [PATCH] [CI] Signed-off-by: turuslan --- .github/workflows/autotests.yml | 48 ---- .github/workflows/test.yml | 366 +-------------------------- .github/workflows/zombie-builder.yml | 35 --- housekeeping/make_build.sh | 1 - 4 files changed, 12 insertions(+), 438 deletions(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index fd520d5134..e69de29bb2 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -1,48 +0,0 @@ -# -# Copyright Quadrivium LLC -# All Rights Reserved -# SPDX-License-Identifier: Apache-2.0 -# - -name: Autotests - -on: - repository_dispatch: - types: [ autotests ] - -jobs: - autotests: - runs-on: ubuntu-latest - container: soramitsu/kagome-autotests:latest - steps: - - name: Set commit status as pending - uses: myrotvorets/set-commit-status-action@7d7fe9a321fbe56a1339449c19422e95a025c65a - with: - token: ${{ secrets.GITHUB_TOKEN }} - status: pending - context: Autotests - sha: ${{ github.sha }} - - name: Run autotests - working-directory: /kagome-autotests - run: python3 -m pytest - shell: sh - - name: Set commit status as failure - if: failure() - uses: myrotvorets/set-commit-status-action@7d7fe9a321fbe56a1339449c19422e95a025c65a - with: - token: ${{ secrets.GITHUB_TOKEN }} - status: failure - context: Autotests - sha: ${{ github.sha }} - - name: Set commit status as success - if: success() - uses: myrotvorets/set-commit-status-action@7d7fe9a321fbe56a1339449c19422e95a025c65a - with: - token: ${{ secrets.GITHUB_TOKEN }} - status: success - context: Autotests - sha: ${{ github.sha }} - env: - KAGOME_S1: ${{ secrets.KAGOME_S1 }} - SUBSTRATE_S2: ${{ secrets.SUBSTRATE_S2 }} - ALICE_ED: ${{ secrets.ALICE_ED }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c20e14f18..5712172d79 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,361 +37,19 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: Setup cmake - uses: jwlawson/actions-setup-cmake@187efd9581ed20ee4e03c0dfb9ac2cd5896c4835 - with: - cmake-version: '3.25.0' - name: install - run: ./housekeeping/macos/dependency.sh + run: + python3 -m venv venv + which python python3 + source venv/bin/activate + which python python3 + ./housekeeping/macos/dependency.sh - name: build - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake + run: + source venv/bin/activate + export HUNTER_PYTHON_LOCATION=$VIRTUAL_ENV + ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake env: KAGOME_IN_DOCKER: 0 - Linux: - if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} - strategy: - fail-fast: false - matrix: - options: - - name: "Linux: clang-15 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake - - name: "${{ matrix.options.name }}" - runs-on: ubuntu-latest - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: df du - run: | - df -m || true - du -hd1 /__w /github || true - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - - name: df du - if: ${{ always() }} - run: | - df -m || true - du -hd1 /__w /github || true - - Linux-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' ) }} - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Linux: gcc-12 ASAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON - - name: "Self-hosted: Linux: clang-15 TSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON - - name: "Self-hosted: Linux: clang-15 UBSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON - - name: "Self-hosted: Linux: clang-15 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake - - name: "${{ matrix.options.name }}" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: df du - run: | - df -m || true - du -hd1 /__w /github || true - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - - name: df du - if: ${{ always() }} - run: | - df -m || true - du -hd1 /__w /github || true - - Linux-self-hosted-PR: - if: ${{ contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Linux: gcc-12 ASAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON - - name: "Self-hosted: Linux: clang-15 TSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON - - name: "Self-hosted: Linux: clang-15 UBSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON - - name: "Self-hosted: Linux: clang-15 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake - - name: "${{ matrix.options.name }}" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: df du - run: | - df -m || true - du -hd1 /__w /github || true - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - - name: df du - if: ${{ always() }} - run: | - df -m || true - du -hd1 /__w /github || true - - clang-tidy: - name: "Linux: clang-tidy" - runs-on: ubuntu-latest - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - with: - fetch-depth: 0 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: clang-tidy - env: - # build only generated files, so clang-tidy will work correctly - BUILD_FINAL_TARGET: generated - run: | - ./housekeeping/make_build.sh - ./housekeeping/clang-tidy-diff.sh - - coverage-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} - name: "Self-hosted: Linux: gcc-12 coverage/sonar" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: df du - run: | - df -m || true - du -hd1 /__w /github || true - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: makeBuild - env: - BUILD_FINAL_TARGET: ctest_coverage - run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON - - name: df du - if: ${{ always() }} - run: | - df -m || true - du -hd1 /__w /github || true - - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - name: Submit Coverage - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi - - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - name: Sonar - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH_NAME: ${{ github.ref }} - run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi - - Minideb: - if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} - strategy: - fail-fast: false - matrix: - options: - - name: "Minideb: Build Debug" - build-type: "Debug" - - name: "Minideb: Build Release" - build-type: "Release" - name: "${{ matrix.options.name }}" - runs-on: ubuntu-latest - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: "${{ matrix.options.name }}" - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - - Minideb-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Minideb: Build Debug" - build-type: "Debug" - - name: "Self-hosted: Minideb: Build Release" - build-type: "Release" - name: "${{ matrix.options.name }}" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: "${{ matrix.options.name }}" - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - - Push-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' ) }} - needs: [clang-tidy, MacOS, Minideb-self-hosted] - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Push Debug Docker image" - build-type: "Debug" - - name: "Self-hosted: Push Release Docker image" - build-type: "Release" - - name: "Self-hosted: Push RelWithDebInfo Docker image" - build-type: "RelWithDebInfo" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - run: git config --global --add safe.directory /__w/kagome/kagome - - run: git fetch --prune --unshallow - - name: build - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 - with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: docker pack and push - env: - VERSION: ${{ github.ref }} - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome/build_and_push.sh - - Push-self-hosted-PR: - if: ${{ contains( github.event.pull_request.labels.*.name, 'Non-master push') }} - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Push Debug Docker image" - build-type: "Debug" - - name: "Self-hosted: Push Release Docker image" - build-type: "Release" - - name: "Self-hosted: Push RelWithDebInfo Docker image" - build-type: "RelWithDebInfo" - runs-on: [ self-hosted ] - container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - run: git config --global --add safe.directory /__w/kagome/kagome - - run: git fetch --prune --unshallow - - name: build - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 - with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: docker pack and push - env: - VERSION: ${{ github.ref }} - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome/build_and_push.sh - - zombie-tests: - strategy: - fail-fast: false - matrix: - options: - - name: "Parachains smoke test" - test: "test0001" - - name: "Parachains smoke test kagome" - test: "test0001_kagome" - - name: "Parachains upgrade smoke test" - test: "test0002" - - name: "Parachains upgrade smoke test kagome" - test: "test0002_kagome" - - name: "Parachains smoke test cumulus" - test: "test0003" - - name: "Parachains smoke test cumulus kagome" - test: "test0003_kagome" - - name: "Runtime upgrade" - test: "test0004" - - name: "Runtime upgrade kagome" - test: "test0004_kagome" - - name: "Dispute valid block" - test: "test0005" - - name: "Parachains disputes" - test: "test0006" - - name: "Deregister register validator smoke" - test: "test0008" - - name: "Basic warp sync" - test: "test0009" - - name: "Validators warp sync" - test: "test0010" - - name: "Block building warp sync" - test: "test0011" - runs-on: [ self-hosted ] - needs: [Push-self-hosted] - steps: - - name: Set owner of working dir recurively - run: sudo chown -R $(whoami) . - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 - - name: build zombie-tester - working-directory: ./zombienet/docker - run: make tester - - name: Run test - working-directory: ./zombienet/docker - run: make ${{ matrix.options.test }} - - Trigger_CD: - if: ${{ github.ref == 'refs/heads/master' }} - needs: [Push-self-hosted] - runs-on: ubuntu-latest - steps: - - name: trigger cd job - run: | - curl -fI "${{ secrets.JENKINS_URL }}kagome-test&token=${{ secrets.JENKINS_TOKEN }}" - curl -fI "${{ secrets.JENKINS_URL }}polkadot-test&token=${{ secrets.JENKINS_TOKEN }}" + - name: fail + run: false diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml index 65c489896f..e69de29bb2 100644 --- a/.github/workflows/zombie-builder.yml +++ b/.github/workflows/zombie-builder.yml @@ -1,35 +0,0 @@ -# -# Copyright Quadrivium LLC -# All Rights Reserved -# SPDX-License-Identifier: Apache-2.0 -# - -name : Zombie Builder - -# Update zombie builder image each Sunday night -on: -# schedule: -# - cron: '0 2 * * 0' - workflow_dispatch: - -jobs: - build_and_push_zombie_builder: - runs-on: [ kagome, self-hosted, linux ] - container: soramitsu/kagome-dev@sha256:a3153bda78e15337fc38b6800f508e8c3b27758cba6319e60e7c805b06e071f6 - steps: - - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6 -# - name: Install dependencies -# working-directory: ./zombienet/docker -# run: | -# pip install -r ./requirements.txt - - name: build zombie-builder - working-directory: ./zombienet/docker - run: make builder_last - - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 - with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Push zombie-builder docker image - working-directory: ./zombienet/docker - run: make builder_push diff --git a/housekeeping/make_build.sh b/housekeeping/make_build.sh index 9cfa23d142..608bad64b9 100755 --- a/housekeeping/make_build.sh +++ b/housekeeping/make_build.sh @@ -27,4 +27,3 @@ cmake . -B${BUILD_DIR} "$@" -DBACKWARD=OFF if [ "$BUILD_FINAL_TARGET" != "generated" ] ; then cmake --build "${BUILD_DIR}" -- -j${BUILD_THREADS} fi -cmake --build "${BUILD_DIR}" --target "${BUILD_FINAL_TARGET}"