Skip to content
Main Build Pipeline

pvf secure clone #7959

Sign in to view logs

pvf secure clone

pvf secure clone #7959

Workflow file for this run

.github/workflows/test.yml at b445bcb
#
# Copyright Quadrivium LLC
# All Rights Reserved
# SPDX-License-Identifier: Apache-2.0
#
name: Main Build Pipeline
on:
push:
branches: [ master ]
tags: [ '*' ]
pull_request:
workflow_dispatch:
inputs:
run_all_steps:
description: 'Run all steps'
required: true
default: 'false'
type: 'choice'
options:
- 'true'
- 'false'
use_cache:
description: 'Use cache for build'
required: true
default: 'true'
type: 'choice'
options:
- 'true'
- 'false'
builder_latest_tag:
description: 'Custom Builder tag'
default: 'latest'
required: false
type: 'string'
zombie_tester_latest_tag:
description: 'Custom Zombie Tester tag'
default: 'latest'
required: false
type: 'string'
env:
BUILD_DIR: build
KAGOME_IN_DOCKER: 1
CTEST_OUTPUT_ON_FAILURE: 1
GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }}
GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }}
# Change all container: values to this variable once this feature is available
# https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6
CACHE_VERSION: v01
CACHE_PATHS: |
~/Library/Caches/pip
~/.cargo
~/.ccache
~/.hunter
~/.rustup
DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
PROJECT_ID: ${{ secrets.GCP_PROJECT }}
CACHE_PATH: ./build/cache
USE_CACHE: ${{ github.event.inputs.use_cache || 'true' }}
DEFAULT_BUILD_TYPE: Release
GIT_REF_NAME: ${{ github.ref_name }}
IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }}
BUILDER_LATEST_TAG: ${{ github.event.inputs.builder_latest_tag || 'latest' }}
TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
jobs:
MacOS:
runs-on: macos-15
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
options:
- name: "MacOS: Build Debug"
build_type: "Debug"
wasm_compiler: "WasmEdge"
- name: "MacOS: Build Release"
build_type: "Release"
wasm_compiler: "WasmEdge"
- name: "MacOS WAVM: Build Debug"
build_type: "Debug"
wasm_compiler: "WAVM"
name: "${{ matrix.options.name }}"
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
if: ${{ env.USE_CACHE == 'true' }}
with:
path: ${{ env.CACHE_PATHS }}
key: ${{ github.job }}-${{ env.CACHE_VERSION }}
- name: install
run: ./housekeeping/macos/dependency.sh
env:
KAGOME_MAC_CI: 1
- name: build
run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=${{ matrix.options.wasm_compiler }} -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake
env:
KAGOME_IN_DOCKER: 0
KAGOME_MAC_CI: 1
Linux:
if: false # Need to fix
# ${{
# !(github.ref == 'refs/heads/master' ||
# startsWith( github.ref, 'refs/tags/' )) }}
strategy:
fail-fast: false
matrix:
options:
- name: "Linux: clang-19 External Project"
run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake
name: "${{ matrix.options.name }}"
runs-on: ubuntu-24.04
timeout-minutes: 120
container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
if: ${{ env.USE_CACHE == 'true' }}
with:
path: ${{ env.CACHE_PATHS }}
key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
- name: df du
run: |
df -m || true
du -hd1 /__w /github || true
- name: Install mold
run: ./housekeeping/ci_install_mold.sh --make-default
- name: "${{ matrix.options.name }}"
run: "${{ matrix.options.run }}"
- name: df du
if: ${{ always() }}
run: |
df -m || true
du -hd1 /__w /github || true
coverage-self-hosted:
if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }}
name: "Self-hosted: Linux: gcc-13 coverage/sonar"
runs-on: [ actions-runner-controller ]
timeout-minutes: 120
container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
if: ${{ env.USE_CACHE == 'true' }}
with:
path: ${{ env.CACHE_PATHS }}
key: ${{ github.job }}-${{ env.CACHE_VERSION }}
- name: Install mold
run: ./housekeeping/ci_install_mold.sh --make-default
- name: makeBuild
env:
BUILD_FINAL_TARGET: ctest_coverage
run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON
- if: ${{
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository }}
name: Submit Coverage
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi
- if: ${{
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository }}
name: Sonar
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BRANCH_NAME: ${{ github.ref }}
run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi
kagome_dev_docker_build_sanitizers:
strategy:
fail-fast: false
matrix:
options:
- name: "Self-hosted: Linux: gcc-13 ASAN"
params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON
- name: "Self-hosted: Linux: clang-19 TSAN WAVM"
params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DTSAN=ON
- name: "Self-hosted: Linux: clang-19 UBSAN"
params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON
name: "${{ matrix.options.name }}"
runs-on: [ actions-runner-controller ]
timeout-minutes: 180
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Get master branch"
if: github.ref != 'refs/heads/master'
run: git fetch origin master:master || true
- name: "Get commit version"
working-directory: ./housekeeping/docker/kagome-dev
run: make get_versions
- name: "Check version"
working-directory: ./housekeeping/docker/kagome-dev
run: |
SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV
- name: "Cache dependencies"
id: cache-restore
if: ${{ env.USE_CACHE == 'true' }}
uses: actions/cache/restore@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-${{ env.short_commit_hash }}
restore-keys: |
${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-
- name: "Build target"
working-directory: ./housekeeping/docker/kagome-dev
run:
make kagome_dev_docker_build_sanitizers \
GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
CI="true"
SAN_PARAMS="${{ matrix.options.params }}"
- name: "Cleaning cache"
run: |
find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \;
- name: "Always Save Cache"
id: cache-save
if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')
uses: actions/cache/save@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ steps.cache-restore.outputs.cache-primary-key }}
kagome_dev_docker_build_tidy:
runs-on: ubuntu-24.04
timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208
name: "Linux: clang-tidy"
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Get master branch"
if: github.ref != 'refs/heads/master'
run: git fetch origin master:master || true
- name: "Get commit version"
working-directory: ./housekeeping/docker/kagome-dev
run: make get_versions
- name: "Check version"
working-directory: ./housekeeping/docker/kagome-dev
run: |
SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV
- name: "Cache dependencies"
id: cache-restore
if: ${{ env.USE_CACHE == 'true' }}
uses: actions/cache/restore@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-${{ env.short_commit_hash }}
restore-keys: |
${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-
- name: "Build target"
working-directory: ./housekeeping/docker/kagome-dev
run:
make kagome_dev_docker_build_tidy \
GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
CI="true"
- name: "Cleaning cache"
run: |
find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \;
- name: "Always Save Cache"
id: cache-save
if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')
uses: actions/cache/save@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ steps.cache-restore.outputs.cache-primary-key }}
kagome_dev_docker_build:
runs-on: [ actions-runner-controller ]
timeout-minutes: 180
if: ${{
github.ref == 'refs/heads/master' ||
startsWith( github.ref, 'refs/tags/') ||
contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} ||
github.event.inputs.run_all_steps == 'true' }}
strategy:
fail-fast: false
matrix:
options:
- name: "Self-hosted docker: Build Debug"
build_type: "Debug"
- name: "Self-hosted docker: Build Release"
build_type: "Release"
- name: "Self-hosted docker: Build RelWithDebInfo"
build_type: "RelWithDebInfo"
name: "${{ matrix.options.name }}"
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Get master branch"
if: github.ref != 'refs/heads/master'
run: git fetch origin master:master || true
- name: "Authenticate with Google Cloud"
uses: 'google-github-actions/auth@v2'
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
- name: "Set up Cloud SDK"
uses: 'google-github-actions/setup-gcloud@v2'
- name: "Configure Docker for GCR"
run: |
gcloud auth configure-docker --quiet
gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
- name: "Get commit version"
working-directory: ./housekeeping/docker/kagome-dev
run: make get_versions
- name: "Check package version"
working-directory: ./housekeeping/docker/kagome-dev
run: |
ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //')
REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //')
SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ matrix.options.build_type }}"
echo "PACKAGE_VERSION=${PACKAGE_VERSION}"
echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV
echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV
gcloud config set artifacts/repository $ARTIFACTS_REPO
gcloud config set artifacts/location $REGION
PACKAGE_EXIST=$(gcloud artifacts versions list --package=kagome-dev --format=json | jq -e ".[] | select(.name | endswith(\"${PACKAGE_VERSION}\"))" > /dev/null && echo "True" || echo "False")
echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV
- name: "Cache dependencies"
if: ${{ env.package_exist == 'False' && env.USE_CACHE == 'true' }}
id: cache-restore
uses: actions/cache/restore@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}-${{ env.short_commit_hash }}
restore-keys: |
${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}-
- name: "Build target"
if: env.package_exist == 'False'
working-directory: ./housekeeping/docker/kagome-dev
run: make kagome_dev_docker_build \
DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
GIT_REF_NAME=${{ env.GIT_REF_NAME }} \
BUILD_TYPE=${{ matrix.options.build_type }}
- name: "Push Kagome APT Package"
if: env.package_exist == 'False'
working-directory: ./housekeeping/docker/kagome-dev
run: |
SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
make upload_apt_package \
BUILD_TYPE=${{ matrix.options.build_type }}
- name: "Set up Docker Buildx"
if: env.package_exist == 'False'
uses: docker/[email protected]
- name: "Build Docker Image"
if: env.package_exist == 'False'
working-directory: ./housekeeping/docker/kagome-dev
run: make kagome_image_build \
KAGOME_PACKAGE_VERSION=${{ env.package_version }} \
DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
BUILD_TYPE=${{ matrix.options.build_type }}
- name: "Push Docker Image to GCR"
if: env.package_exist == 'False'
working-directory: ./housekeeping/docker/kagome-dev
run: make kagome_image_push \
DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
BUILD_TYPE=${{ matrix.options.build_type }} \
GIT_REF_NAME=${{ env.GIT_REF_NAME }} \
IS_MAIN_OR_TAG=${{ env.IS_MAIN_OR_TAG }}
- name: "Login to Docker Hub"
uses: docker/login-action@v3
if: ${{
(env.package_exist == 'False') &&
(matrix.options.build_type == 'Release') }}
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: "Push Docker Image to Docker Hub"
if: ${{
(env.package_exist == 'False') &&
(matrix.options.build_type == 'Release') }}
working-directory: ./housekeeping/docker/kagome-dev
run: make kagome_image_push_dockerhub \
DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
BUILD_TYPE=${{ matrix.options.build_type }} \
GIT_REF_NAME=${{ env.GIT_REF_NAME }} \
IS_MAIN_OR_TAG=${{ env.IS_MAIN_OR_TAG }}
- name: "Cleaning cache"
if: env.package_exist == 'False'
run: |
find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \;
- name: "Always Save Cache"
id: cache-save
if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')
uses: actions/cache/save@v4
with:
path: ${{ env.CACHE_PATH }}
key: ${{ steps.cache-restore.outputs.cache-primary-key }}
- name: "Versions Info"
if: always()
run: |
echo "Kagome Package Version: kagome-dev ${{ env.package_version }}"
echo "Kagome Images: "
cat /tmp/docker_image.env || true
cat /tmp/dockerhub_image.env || true
Zombie-Tests:
runs-on: [ actions-runner-controller ]
timeout-minutes: 60
needs: [kagome_dev_docker_build]
if: ${{
github.ref == 'refs/heads/master' ||
startsWith( github.ref, 'refs/tags/') ||
github.event.inputs.run_all_steps == 'true' }}
strategy:
fail-fast: false
matrix:
options:
- name: "PVF preparation & execution time"
test: "test-polkadot-functional-0001-parachains-pvf"
# - name: "Disputes initiation, conclusion and lag"
# test: "test-polkadot-functional-0002-parachains-disputes"
- name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute"
test: "test-polkadot-functional-0003-beefy-and-mmr"
# - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block"
# test: "test-polkadot-functional-0004-parachains-garbage-candidate"
# - name: "Past-session dispute slashing"
# test: "test-polkadot-functional-0005-parachains-disputes-past-session"
# - name: "Test if parachains make progress with most of approvals being tranch0"
# test: "test-polkadot-functional-0006-parachains-max-tranche0"
- name: "Test if disputes triggered on finalized blocks within scope always end as valid"
test: "test-polkadot-functional-0007-dispute-freshly-finalized"
- name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded"
test: "test-polkadot-functional-0008-dispute-old-finalized"
- name: "Approval voting coalescing does not lag finality"
test: "test-polkadot-functional-0009-approval-voting-coalescing"
# - name: "Test validator disabling effects"
# test: "test-polkadot-functional-0010-validator-disabling"
- name: "Test we are producing blocks at 6 seconds clip"
test: "test-polkadot-functional-0011-async-backing-6-seconds-rate"
- name: "Systematic chunk recovery is used if the chunk mapping feature is enabled."
test: "test-polkadot-functional-0013-systematic-chunk-recovery"
- name: "Warp sync from polkadot node"
test: "test-custom-0001-validators-warp-sync"
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Get master branch"
if: github.ref != 'refs/heads/master'
run: git fetch origin master:master || true
# - name: Create and enable swap
# run: |
# free -h
# df -h
# sudo fallocate -l 4G /swapfile
# sudo chmod 600 /swapfile
# sudo mkswap /swapfile
# sudo swapon /swapfile
# sudo swapon --show
# free -h
# df -h
- name: "Authenticate with Google Cloud"
uses: 'google-github-actions/auth@v2'
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
- name: "Set up Cloud SDK"
uses: 'google-github-actions/setup-gcloud@v2'
- name: "Configure Docker for GCR"
run: |
gcloud auth configure-docker --quiet
gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
- name: "Get commit version"
working-directory: ./housekeeping/docker/kagome-dev
run: make get_versions
- name: "Get Package Version"
working-directory: ./housekeeping/docker/kagome-dev
run: |
ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //')
REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //')
SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ env.DEFAULT_BUILD_TYPE }}"
echo "PACKAGE_VERSION=${PACKAGE_VERSION}"
echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV
- name: "Get Zombie Tester Version"
run: |
ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${{ env.TESTER_LATEST_TAG }}
echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}"
echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV
- name: "Run test"
working-directory: ./zombienet/docker
run: |
make ${{ matrix.options.test }} \
ZOMBIE_TESTER_IMAGE=${{ env.zombie_tester_image }} \
KAGOME_PACKAGE_VERSION=${{ env.package_version }}
free -h
df -h
- name: "Upload test logs as artifact"
if: always()
uses: actions/upload-artifact@v4
with:
name: logs-${{ matrix.options.test }}
path: /tmp/test_logs
- name: "Versions Info"
if: always()
run: |
echo "Kagome Package Version: kagome-dev ${{ env.package_version }}"
echo "Zombie Tester Image: ${{ env.zombie_tester_image }}"
echo "Zombie Tests Versions: "
cat /tmp/versions.env || true
- name: "Show system info"
if: always()
run: |
free -h
df -h