Skip to content

Commit

Permalink
Merge branch 'master' into cl/pallet-offences-bench-v2
Browse files Browse the repository at this point in the history
  • Loading branch information
re-gius authored Dec 2, 2024
2 parents 9acdfec + c092133 commit 2a8c1b5
Show file tree
Hide file tree
Showing 1,047 changed files with 62,931 additions and 35,418 deletions.
8 changes: 7 additions & 1 deletion .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ retries = 5
# The number of threads to run tests with. Supported values are either an integer or
# the string "num-cpus". Can be overridden through the `--test-threads` option.
# test-threads = "num-cpus"

test-threads = 20

# The number of threads required for each test. This is generally used in overrides to
Expand Down Expand Up @@ -124,3 +123,10 @@ serial-integration = { max-threads = 1 }
[[profile.default.overrides]]
filter = 'test(/(^ui$|_ui|ui_)/)'
test-group = 'serial-integration'

# Running eth-rpc tests sequentially
# These tests rely on a shared resource (the RPC and Node)
# and would cause race conditions due to transaction nonces if run in parallel.
[[profile.default.overrides]]
filter = 'package(pallet-revive-eth-rpc) and test(/^tests::/)'
test-group = 'serial-integration'
2 changes: 1 addition & 1 deletion .config/zepter.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ workflows:
]
# The umbrella crate uses more features, so we to check those too:
check_umbrella:
- [ $check.0, '--features=serde,experimental,riscv,runtime,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
- [ $check.0, '--features=serde,experimental,runtime,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
# Same as `check_*`, but with the `--fix` flag.
default:
- [ $check.0, '--fix' ]
Expand Down
28 changes: 28 additions & 0 deletions .github/actions/workflow-stopper/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: "stop all workflows"
description: "Action stops all workflows in a PR to save compute resources."
inputs:
app-id:
description: "App id"
required: true
app-key:
description: "App token"
required: true
runs:
using: "composite"
steps:
- name: Worfklow stopper - Generate token
uses: actions/create-github-app-token@v1
id: app-token
with:
app-id: ${{ inputs.app-id }}
private-key: ${{ inputs.app-key }}
owner: "paritytech"
repositories: "workflow-stopper"
- name: Workflow stopper - Stop all workflows
uses: octokit/[email protected]
with:
route: POST /repos/paritytech/workflow-stopper/actions/workflows/stopper.yml/dispatches
ref: main
inputs: '${{ format(''{{ "github_sha": "{0}", "github_repository": "{1}", "github_ref_name": "{2}", "github_workflow_id": "{3}", "github_job_name": "{4}" }}'', github.event.pull_request.head.sha, github.repository, github.ref_name, github.run_id, github.job) }}'
env:
GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
2 changes: 1 addition & 1 deletion .github/env
Original file line number Diff line number Diff line change
@@ -1 +1 @@
IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"
36 changes: 36 additions & 0 deletions .github/scripts/check-missing-readme-generation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash
echo "Running script relative to `pwd`"
# Find all README.docify.md files
DOCIFY_FILES=$(find . -name "README.docify.md")

# Initialize a variable to track directories needing README regeneration
NEED_REGENERATION=""

for file in $DOCIFY_FILES; do
echo "Processing $file"

# Get the directory containing the docify file
DIR=$(dirname "$file")

# Go to the directory and run cargo build
cd "$DIR"
cargo check --features generate-readme || { echo "Readme generation for $DIR failed. Ensure the crate compiles successfully and has a `generate-readme` feature which guards markdown compilation in the crate as follows: https://docs.rs/docify/latest/docify/macro.compile_markdown.html#conventions." && exit 1; }

# Check if README.md has any uncommitted changes
git diff --exit-code README.md

if [ $? -ne 0 ]; then
echo "Error: Found uncommitted changes in $DIR/README.md"
NEED_REGENERATION="$NEED_REGENERATION $DIR"
fi

# Return to the original directory
cd - > /dev/null
done

# Check if any directories need README regeneration
if [ -n "$NEED_REGENERATION" ]; then
echo "The following directories need README regeneration:"
echo "$NEED_REGENERATION"
exit 1
fi
26 changes: 13 additions & 13 deletions .github/scripts/cmd/test_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"path": "substrate/frame",
"header": "substrate/HEADER-APACHE2",
"template": "substrate/.maintain/frame-weight-template.hbs",
"bench_features": "runtime-benchmarks,riscv",
"bench_features": "runtime-benchmarks",
"bench_flags": "--flag1 --flag2"
},
{
Expand Down Expand Up @@ -67,7 +67,7 @@ def setUp(self):
self.patcher6 = patch('importlib.util.spec_from_file_location', return_value=MagicMock())
self.patcher7 = patch('importlib.util.module_from_spec', return_value=MagicMock())
self.patcher8 = patch('cmd.generate_prdoc.main', return_value=0)

self.mock_open = self.patcher1.start()
self.mock_json_load = self.patcher2.start()
self.mock_parse_args = self.patcher3.start()
Expand Down Expand Up @@ -101,27 +101,27 @@ def test_bench_command_normal_execution_all_runtimes(self):
clean=False,
image=None
), [])

self.mock_popen.return_value.read.side_effect = [
"pallet_balances\npallet_staking\npallet_something\n", # Output for dev runtime
"pallet_balances\npallet_staking\npallet_something\n", # Output for westend runtime
"pallet_staking\npallet_something\n", # Output for rococo runtime - no pallet here
"pallet_balances\npallet_staking\npallet_something\n", # Output for asset-hub-westend runtime
"./substrate/frame/balances/Cargo.toml\n", # Mock manifest path for dev -> pallet_balances
]

with patch('sys.exit') as mock_exit:
import cmd
cmd.main()
mock_exit.assert_not_called()

expected_calls = [
# Build calls
call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"),
call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks"),
call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"),
call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),

call(get_mock_bench_output(
runtime='kitchensink',
pallets='pallet_balances',
Expand Down Expand Up @@ -162,7 +162,7 @@ def test_bench_command_normal_execution(self):
self.mock_popen.return_value.read.side_effect = [
"pallet_balances\npallet_staking\npallet_something\n", # Output for westend runtime
]

with patch('sys.exit') as mock_exit:
import cmd
cmd.main()
Expand All @@ -171,7 +171,7 @@ def test_bench_command_normal_execution(self):
expected_calls = [
# Build calls
call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),

# Westend runtime calls
call(get_mock_bench_output(
runtime='westend',
Expand Down Expand Up @@ -205,7 +205,7 @@ def test_bench_command_normal_execution_xcm(self):
self.mock_popen.return_value.read.side_effect = [
"pallet_balances\npallet_staking\npallet_something\npallet_xcm_benchmarks::generic\n", # Output for westend runtime
]

with patch('sys.exit') as mock_exit:
import cmd
cmd.main()
Expand All @@ -214,7 +214,7 @@ def test_bench_command_normal_execution_xcm(self):
expected_calls = [
# Build calls
call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),

# Westend runtime calls
call(get_mock_bench_output(
runtime='westend',
Expand All @@ -241,7 +241,7 @@ def test_bench_command_two_runtimes_two_pallets(self):
"pallet_staking\npallet_balances\n", # Output for westend runtime
"pallet_staking\npallet_balances\n", # Output for rococo runtime
]

with patch('sys.exit') as mock_exit:
import cmd
cmd.main()
Expand Down Expand Up @@ -309,7 +309,7 @@ def test_bench_command_one_dev_runtime(self):

expected_calls = [
# Build calls
call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"),
call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks"),
# Westend runtime calls
call(get_mock_bench_output(
runtime='kitchensink',
Expand Down Expand Up @@ -429,4 +429,4 @@ def test_prdoc_command(self, mock_system, mock_parse_args):
self.mock_generate_prdoc_main.assert_called_with(mock_parse_args.return_value[0])

if __name__ == '__main__':
unittest.main()
unittest.main()
36 changes: 33 additions & 3 deletions .github/scripts/common/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -237,15 +237,44 @@ fetch_release_artifacts() {
popd > /dev/null
}

# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# Fetch deb package from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form paritytech/polkadot
fetch_release_artifacts_from_s3() {
fetch_debian_package_from_s3() {
BINARY=$1
echo "Version : $VERSION"
echo "Repo : $REPO"
echo "Binary : $BINARY"
echo "Tag : $RELEASE_TAG"
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"

URL_BASE=$(get_s3_url_base $BINARY)
echo "URL_BASE=$URL_BASE"

URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb

mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null

echo "Fetching deb package..."

echo "Fetching %s" "$URL"
curl --progress-bar -LO "$URL" || echo "Missing $URL"

pwd
ls -al --color
popd > /dev/null

}

# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form paritytech/polkadot
fetch_release_artifacts_from_s3() {
BINARY=$1
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"

Expand Down Expand Up @@ -306,9 +335,10 @@ function import_gpg_keys() {
EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
PARITY_RELEASES="90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE"
PARITY_RELEASES_SIGN_COMMITS="D8018FBB3F534D866A45998293C5FB5F6A367B51"

echo "Importing GPG keys from $GPG_KEYSERVER"
for key in $SEC $EGOR $MORGAN $PARITY_RELEASES; do
for key in $SEC $EGOR $MORGAN $PARITY_RELEASES $PARITY_RELEASES_SIGN_COMMITS; do
(
echo "Importing GPG key $key"
gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
Expand Down
3 changes: 1 addition & 2 deletions .github/scripts/release/build-deb.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ cargo install --version 2.7.0 cargo-deb --locked -q
echo "Using cargo-deb v$(cargo-deb --version)"
echo "Building a Debian package for '$PRODUCT' in '$PROFILE' profile"

# we need to start the custom version with a didgit as requires it cargo-deb
cargo deb --profile $PROFILE --no-strip --no-build -p $PRODUCT --deb-version 1-$VERSION
cargo deb --profile $PROFILE --no-strip --no-build -p $PRODUCT --deb-version $VERSION

deb=target/debian/$PRODUCT_*_amd64.deb

Expand Down
4 changes: 3 additions & 1 deletion .github/scripts/release/build-linux-release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
# This is used to build our binaries:
# - polkadot
# - polkadot-parachain
# - polkadot-omni-node
#
# set -e

BIN=$1
Expand All @@ -21,7 +23,7 @@ time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PAC
echo "Artifact target: $ARTIFACTS"

cp ./target/$PROFILE/$BIN "$ARTIFACTS"
pushd "$ARTIFACTS" > /dev/nul
pushd "$ARTIFACTS" > /dev/null
sha256sum "$BIN" | tee "$BIN.sha256"

EXTRATAG="$($ARTIFACTS/$BIN --version |
Expand Down
37 changes: 37 additions & 0 deletions .github/scripts/release/build-macos-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/usr/bin/env bash

# This is used to build our binaries:
# - polkadot
# - polkadot-parachain
# - polkadot-omni-node
# set -e

BIN=$1
PACKAGE=${2:-$BIN}

PROFILE=${PROFILE:-production}
# parity-macos runner needs a path where it can
# write, so make it relative to github workspace.
ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")

echo "Artifacts will be copied into $ARTIFACTS"
mkdir -p "$ARTIFACTS"

git log --pretty=oneline -n 1
time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE

echo "Artifact target: $ARTIFACTS"

cp ./target/$PROFILE/$BIN "$ARTIFACTS"
pushd "$ARTIFACTS" > /dev/null
sha256sum "$BIN" | tee "$BIN.sha256"

EXTRATAG="$($ARTIFACTS/$BIN --version |
sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"

EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"

echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
echo -n ${VERSION} > "$ARTIFACTS/VERSION"
echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
39 changes: 39 additions & 0 deletions .github/scripts/release/distributions
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
Origin: Parity
Label: Parity
Codename: release
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity Staging
Codename: staging
Architectures: amd64
Components: main
Description: Staging distribution for Parity Technologies Ltd. packages
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2407
Codename: stable2407
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2409
Codename: stable2409
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2412
Codename: stable2412
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
Loading

0 comments on commit 2a8c1b5

Please sign in to comment.