Merge branch 'master' into people-chain-integration-tests

.config/nextest.toml

@@ -21,7 +21,6 @@ retries = 5
 # The number of threads to run tests with. Supported values are either an integer or
 # the string "num-cpus". Can be overridden through the `--test-threads` option.
 # test-threads = "num-cpus"
-
 test-threads = 20
 
 # The number of threads required for each test. This is generally used in overrides to

.github/actions/workflow-stopper/action.yml

@@ -0,0 +1,28 @@
+name: "stop all workflows"
+description: "Action stops all workflows in a PR to save compute resources."
+inputs:
+  app-id:
+    description: "App id"
+    required: true
+  app-key:
+    description: "App token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Worfklow stopper - Generate token
+      uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ inputs.app-id }}
+        private-key: ${{ inputs.app-key }}
+        owner: "paritytech"
+        repositories: "workflow-stopper"
+    - name: Workflow stopper - Stop all workflows
+      uses: octokit/[email protected]
+      with:
+        route: POST /repos/paritytech/workflow-stopper/actions/workflows/stopper.yml/dispatches
+        ref: main
+        inputs: '${{ format(''{{ "github_sha": "{0}", "github_repository": "{1}", "github_ref_name": "{2}", "github_workflow_id": "{3}", "github_job_name": "{4}" }}'', github.event.pull_request.head.sha, github.repository, github.ref_name, github.run_id, github.job) }}'
+      env:
+        GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}

.github/scripts/check-missing-readme-generation.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+echo "Running script relative to `pwd`"
+# Find all README.docify.md files
+DOCIFY_FILES=$(find . -name "README.docify.md")
+
+# Initialize a variable to track directories needing README regeneration
+NEED_REGENERATION=""
+
+for file in $DOCIFY_FILES; do
+  echo "Processing $file"
+
+  # Get the directory containing the docify file
+  DIR=$(dirname "$file")
+
+  # Go to the directory and run cargo build
+  cd "$DIR"
+  cargo check --features generate-readme || { echo "Readme generation for $DIR failed. Ensure the crate compiles successfully and has a `generate-readme` feature which guards markdown compilation in the crate as follows: https://docs.rs/docify/latest/docify/macro.compile_markdown.html#conventions." && exit 1; }
+
+  # Check if README.md has any uncommitted changes
+  git diff --exit-code README.md
+
+  if [ $? -ne 0 ]; then
+    echo "Error: Found uncommitted changes in $DIR/README.md"
+    NEED_REGENERATION="$NEED_REGENERATION $DIR"
+  fi
+
+  # Return to the original directory
+  cd - > /dev/null
+done
+
+# Check if any directories need README regeneration
+if [ -n "$NEED_REGENERATION" ]; then
+  echo "The following directories need README regeneration:"
+  echo "$NEED_REGENERATION"
+  exit 1
+fi

.github/scripts/common/lib.sh

@@ -237,15 +237,44 @@ fetch_release_artifacts() {
   popd > /dev/null
 }
 
-# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# Fetch deb package from S3. Assumes the ENV are set:
 # - RELEASE_ID
 # - GITHUB_TOKEN
 # - REPO in the form paritytech/polkadot
-fetch_release_artifacts_from_s3() {
+fetch_debian_package_from_s3() {
   BINARY=$1
   echo "Version    : $VERSION"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
+  echo "Tag        : $RELEASE_TAG"
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
+
+  URL_BASE=$(get_s3_url_base $BINARY)
+  echo "URL_BASE=$URL_BASE"
+
+  URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
+
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
+
+  echo "Fetching deb package..."
+
+  echo "Fetching %s" "$URL"
+  curl --progress-bar -LO "$URL" || echo "Missing $URL"
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+
+}
+
+# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts_from_s3() {
+  BINARY=$1
   OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 

.github/scripts/release/build-linux-release.sh

@@ -3,6 +3,8 @@
 # This is used to build our binaries:
 # - polkadot
 # - polkadot-parachain
+# - polkadot-omni-node 
+#
 # set -e
 
 BIN=$1
@@ -21,7 +23,7 @@ time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PAC
 echo "Artifact target: $ARTIFACTS"
 
 cp ./target/$PROFILE/$BIN "$ARTIFACTS"
-pushd "$ARTIFACTS" > /dev/nul
+pushd "$ARTIFACTS" > /dev/null
 sha256sum "$BIN" | tee "$BIN.sha256"
 
 EXTRATAG="$($ARTIFACTS/$BIN --version |

.github/scripts/release/build-macos-release.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+# parity-macos runner needs a path where it can
+# write, so make it relative to github workspace.
+ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"

.github/scripts/release/distributions

@@ -0,0 +1,39 @@
+Origin: Parity
+Label: Parity
+Codename: release
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity Staging
+Codename: staging
+Architectures: amd64
+Components: main
+Description: Staging distribution for Parity Technologies Ltd. packages
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2407
+Codename: stable2407
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2409
+Codename: stable2409
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2412
+Codename: stable2412
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

.github/scripts/release/release_lib.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Set the new version by replacing the value of the constant given as patetrn
+# Set the new version by replacing the value of the constant given as pattern
 # in the file.
 #
 # input: pattern, version, file
@@ -119,21 +119,23 @@ set_polkadot_parachain_binary_version() {
 
 
 upload_s3_release() {
-  alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
-
-  product=$1
-  version=$2
-
-  echo "Working on product: $product "
-  echo "Working on version: $version "
-
-  echo "Current content, should be empty on new uploads:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize || true
-  echo "Content to be uploaded:"
-  artifacts="artifacts/$product/"
-  ls "$artifacts"
-  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/${version}/"
-  echo "Uploaded files:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize
-  echo "✅ The release should be at https://releases.parity.io/polkadot/${version}"
+    alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+    product=$1
+    version=$2
+    target=$3
+
+    echo "Working on product:  $product "
+    echo "Working on version:  $version "
+    echo "Working on platform: $target "
+
+    echo "Current content, should be empty on new uploads:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
+    echo "Content to be uploaded:"
+    artifacts="artifacts/$product/"
+    ls "$artifacts"
+    aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
+    echo "Uploaded files:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
 }

.github/workflows/build-misc.yml

@@ -16,7 +16,6 @@ permissions:
   contents: read
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
@@ -38,11 +37,18 @@ jobs:
       - name: Build
         env:
           SUBSTRATE_RUNTIME_TARGET: riscv
+        id: required
         run: |
           forklift cargo check -p minimal-template-runtime
           forklift cargo check -p westend-runtime
           forklift cargo check -p rococo-runtime
           forklift cargo check -p polkadot-test-runtime
+      - name: Stop all workflows if failed
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
+        uses: ./.github/actions/workflow-stopper
+        with:
+          app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
+          app-key: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_KEY }}
 
   build-subkey:
     timeout-minutes: 20
@@ -62,9 +68,16 @@ jobs:
       - name: Build
         env:
           SKIP_WASM_BUILD: 1
+        id: required
         run: |
           cd ./substrate/bin/utils/subkey
           forklift cargo build --locked --release
+      - name: Stop all workflows if failed
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
+        uses: ./.github/actions/workflow-stopper
+        with:
+          app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
+          app-key: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_KEY }}
 
   confirm-required-build-misc-jobs-passed:
     runs-on: ubuntu-latest

.github/workflows/check-frame-omni-bencher.yml

@@ -36,9 +36,16 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
+        id: required
         run: |
           forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
           forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
+      - name: Stop all workflows if failed
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
+        uses: ./.github/actions/workflow-stopper
+        with:
+          app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
+          app-key: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_KEY }}
 
   runtime-matrix:
     runs-on: ubuntu-latest
@@ -80,6 +87,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: script
+        id: required
         run: |
           RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
           RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
@@ -90,6 +98,13 @@ jobs:
           cmd="./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 $FLAGS"
           echo "Running command: $cmd"
           eval "$cmd"
+      - name: Stop all workflows if failed
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
+        uses: ./.github/actions/workflow-stopper
+        with:
+          app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
+          app-key: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_KEY }}
+
   confirm-frame-omni-benchers-passed:
     runs-on: ubuntu-latest
     name: All benchmarks passed

.github/workflows/check-runtime-migration.yml

@@ -101,20 +101,29 @@ jobs:
           ./try-runtime create-snapshot --uri ${{ matrix.uri }} snapshot.raw
 
       - name: Build Runtime
+        id: required1
         run: |
           echo "---------- Building ${{ matrix.package }} runtime ----------"
-          time forklift cargo build --release --locked -p ${{ matrix.package }} --features try-runtime -q
+          forklift cargo build --release --locked -p ${{ matrix.package }} --features try-runtime -q
 
       - name: Run Check
+        id: required2
         run: |
           echo "Running ${{ matrix.network }} runtime migration check"
           export RUST_LOG=remote-ext=debug,runtime=debug
 
           echo "---------- Executing on-runtime-upgrade for ${{ matrix.network }} ----------"
-          time ./try-runtime ${{ matrix.command_extra_args }} \
+          ./try-runtime ${{ matrix.command_extra_args }} \
               --runtime ./target/release/wbuild/${{ matrix.package }}/${{ matrix.wasm }} \
               on-runtime-upgrade --disable-spec-version-check --checks=all ${{ matrix.subcommand_extra_args }} snap -p snapshot.raw
           sleep 5
+      - name: Stop all workflows if failed
+        if: ${{ failure() && (steps.required1.conclusion == 'failure' || steps.required2.conclusion == 'failure') }}
+        uses: ./.github/actions/workflow-stopper
+        with:
+          app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
+          app-key: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_KEY }}
+
   # name of this job must be unique across all workflows
   # otherwise GitHub will mark all these jobs as required
   confirm-required-checks-passed:

.github/workflows/check-semver.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TOOLCHAIN: nightly-2024-06-01
+  TOOLCHAIN: nightly-2024-10-19
 
 jobs:
   preflight:
@@ -74,7 +74,7 @@ jobs:
 
       - name: install parity-publish
         # Set the target dir to cache the build.
-        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.8.0 --locked -q
+        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.10.1 --locked -q
 
       - name: check semver
         run: |