Skip to content
Build and test

feat: Create stable memory data (#4524) #3

Sign in to view logs

feat: Create stable memory data (#4524)

feat: Create stable memory data (#4524) #3

Workflow file for this run

.github/workflows/build.yml at 45cf343
name: Build and test
# We use `push` events so that we have the actual commit. In `pull_request`
# events we get a merge commit with main instead. The merge commit can be
# useful to check that the code would pass tests once merged, but here it just
# creates confusion and doesn't add anything since the branch must be up to
# date before merge. It's also nice to have CI running on branches without PRs.
on:
push:
workflow_dispatch:
inputs:
no_cache:
description: 'no-cache'
default: false
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash -euxlo pipefail {0}
jobs:
build:
runs-on: ubuntu-latest-m
timeout-minutes: 45
env:
GH_TOKEN: ${{ github.token }}
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Skip build for testing
# Set to true and set a recent `run_id` below to reuse an existing build
# instead of building.
if: false
id: skip_build
run: |
echo "skip_build=true" >> "$GITHUB_OUTPUT"
mkdir out
# The run ID is the number at the end of a URL like this:
# https://github.com/dfinity/nns-dapp/actions/runs/5801187848
run_id=5801187848
gh run download "$run_id" --dir ./out -n out
- name: Build nns-dapp repo
if: steps.skip_build.outputs.skip_build != 'true'
uses: ./.github/actions/build_nns_dapp
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: 'Upload nns-dapp wasm module'
uses: actions/upload-artifact@v4
with:
name: nns-dapp
path: out/nns-dapp.wasm.gz
retention-days: 3
- name: 'Upload nns-dapp test wasm module'
uses: actions/upload-artifact@v4
with:
name: nns-dapp_test
path: out/nns-dapp_test.wasm.gz
retention-days: 3
- name: 'Upload sns_aggregator wasm module'
uses: actions/upload-artifact@v4
with:
name: sns_aggregator
path: out/sns_aggregator.wasm.gz
retention-days: 3
- name: 'Upload sns_aggregator_dev wasm module'
uses: actions/upload-artifact@v4
with:
name: sns_aggregator_dev
path: out/sns_aggregator_dev.wasm.gz
retention-days: 3
- name: 'Upload whole out directory'
uses: actions/upload-artifact@v4
with:
name: out
path: out
retention-days: 3
test-playwright-e2e-shard-1-of-2:
needs: build
runs-on: ubuntu-latest-m
timeout-minutes: 30
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Run Playwright e2e test shard 1/2
uses: ./.github/actions/test-e2e
with:
shard_number: 1
shard_count: 2
test-playwright-e2e-shard-2-of-2:
needs: build
runs-on: ubuntu-latest-m
timeout-minutes: 30
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Run Playwright e2e test shard 2/2
uses: ./.github/actions/test-e2e
with:
shard_number: 2
shard_count: 2
test-downgrade-upgrade:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get nns-dapp_test
uses: actions/download-artifact@v4
with:
name: nns-dapp_test
- name: Start snapshot environment
uses: ./.github/actions/start_dfx_snapshot
with:
nns_dapp_wasm: 'nns-dapp_test.wasm.gz'
logfile: 'dfx-test-downgrade-upgrade.log'
- name: Downgrade nns-dapp to prod and upgrade back again
run: ./scripts/nns-dapp/downgrade-upgrade-test -w nns-dapp_test.wasm.gz
- name: Count upgrade cycles
run: scripts/nns-dapp/estimate-upgrade-cycles | tee -a $GITHUB_STEP_SUMMARY
test-upgrade-map:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get nns-dapp_test
uses: actions/download-artifact@v4
with:
name: out
path: out
- name: Install ic-wasm
uses: ./.github/actions/install_ic_wasm
- name: Install dfx
uses: dfinity/setup-dfx@main
- name: Install tools
run: |
sudo apt-get update -yy && sudo apt-get install -yy moreutils && command -v sponge
cargo binstall --no-confirm "idl2json_cli@$(jq -r .defaults.build.config.IDL2JSON_VERSION dfx.json)" && idl2json --version
- name: Start dfx
run: dfx start --clean --background &>test-upgrade-map-dfx.log
- name: Put assets in the default location
run: |
ls
ls out
- name: Upgrade to self, keeping the storage schema as map
run: ./scripts/nns-dapp/migration-test --schema1 Map --schema2 Map --accounts 10000
- name: Upload dfx logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: test-upgrade-map-dfx.log
path: test-upgrade-map-dfx.log
test-upgrade-stable:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get nns-dapp_test
uses: actions/download-artifact@v4
with:
name: out
path: out
- name: Install ic-wasm
uses: ./.github/actions/install_ic_wasm
- name: Install dfx
uses: dfinity/setup-dfx@main
- name: Install tools
run: |
sudo apt-get update -yy && sudo apt-get install -yy moreutils && command -v sponge
cargo binstall --no-confirm "idl2json_cli@$(jq -r .defaults.build.config.IDL2JSON_VERSION dfx.json)" && idl2json --version
- name: Start dfx
run: dfx start --clean --background &>test-upgrade-stable-dfx.log
- name: Downgrade nns-dapp to prod and upgrade back again
run: ./scripts/nns-dapp/migration-test --schema1 AccountsInStableMemory --schema2 AccountsInStableMemory --accounts 1000 --chunk 100
- name: Upload dfx logs
if: failure()
uses: actions/upload-artifact@v3
with:
name: test-upgrade-stable-dfx.log
path: test-upgrade-stable-dfx.log
test-test-account-api:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get nns-dapp_test
uses: actions/download-artifact@v4
with:
name: nns-dapp_test
- name: Start empty nns-dapp
# As long as the snapshot environment can be installed with no accounts, we can use that.
# The snapshot action also installs `idl2json` and `jq`; commands that we will need.
uses: ./.github/actions/start_dfx_snapshot
with:
nns_dapp_wasm: 'nns-dapp_test.wasm.gz'
logfile: 'dfx-test-test-account-api.log'
- name: Check that test accounts can be created and read
run: ./scripts/nns-dapp/test-account.test
test-rest:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 40
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get nns-dapp
uses: actions/download-artifact@v4
with:
name: nns-dapp
- name: Get sns_aggregator
uses: actions/download-artifact@v4
with:
name: sns_aggregator
- name: Get sns_aggregator_dev
uses: actions/download-artifact@v4
with:
name: sns_aggregator_dev
- name: Start snapshot environment
uses: ./.github/actions/start_dfx_snapshot
with:
nns_dapp_wasm: 'nns-dapp.wasm.gz'
sns_aggregator_wasm: 'sns_aggregator_dev.wasm.gz'
logfile: 'dfx-test-rest.log'
- name: Add go and SNS scripts to the path
run: |
echo "$PWD/snsdemo/bin" >> $GITHUB_PATH
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Install command line HTML parser
run: |
go install github.com/ericchiang/pup@latest
pup --version
- name: Verify that arguments are set in index.html
run: |
for ((i=5; i>0; i--)); do
(
timeout 60 curl --fail --silent --retry 10 --connect-timeout 5 "http://$(dfx canister id nns-dapp).localhost:8080/" > index.html
file index.html
< index.html gunzip | pup 'head meta[name="nns-dapp-vars"] json{}' | tee nns_dapp_args_in_page.json
) || { echo "Failed. Retrying..." ; sleep 5 ; continue ; }
break
done
echo "Check a few values:"
for key in data-own-canister-id data-fetch-root-key data-identity-service-url ; do
# Verify that the key is non-trivial:
# `jq -e` returns an error code if the value is missing
# `grep ...` fails if the value is implausibly short.
key="$key" jq -re '.[0][env.key]' nns_dapp_args_in_page.json | grep -E ...
done
- name: Install ic-wasm
uses: ./.github/actions/install_ic_wasm
- name: Check that metadata is present
run: |
scripts/dfx-wasm-metadata-add.test --verbose
- name: Verify that metrics are present
run: scripts/nns-dapp/e2e-test-metrics-present
- name: Release
run: |
for tag in $(git tag --points-at HEAD) ; do
: Creates or updates a release for the tag
if gh release view "$tag"
then gh release upload --repo dfinity/nns-dapp --clobber "$tag" nns-dapp.wasm.gz || true
else gh release create --title "Release for tags/$tag" --draft --notes "Build artefacts from tag: $tag" "$tag" nns-dapp.wasm.gz
fi
: If the tag is for a proposal or nightly, make it public
[[ "$tag" != proposal-* ]] && [[ "$tag" != nightly-* ]] || { echo "Making release public" ; gh release edit "$tag" --draft=false ; }
done
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get the postinstall instruction count
run: |
dfx canister install --upgrade-unchanged nns-dapp --wasm nns-dapp.wasm.gz --mode upgrade --argument "$(cat nns-dapp-arg-local.did)" --yes
postinstall_instructions="$(scripts/backend/get_upgrade_instructions)"
echo "Installation consumed ${postinstall_instructions} instructions."
echo "Cycles consumed are instructions * some factor that depends on subnet. There is no guarantee that that formula will not change."
- name: Stop replica
run: dfx stop
network_independent_wasm:
name: "Same wasms for mainnet and local"
# Note: The dockerfile structure SHOULD guarantee that the network is not used in any Wasm build commands.
# As long as that holds, this test is not needed.
needs: build
runs-on: ubuntu-latest-m
timeout-minutes: 45
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check dockerfile for changes
id: dockerfile_changed
run: |
common_parent_commit="$(git merge-base HEAD origin/main)"
if git diff "$common_parent_commit" Dockerfile | grep -q .
then echo "dockerfile_changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Set up docker buildx
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
uses: docker/setup-buildx-action@v3
- name: Create a blank global config
run: echo "{}" > global-config.json
- name: Build wasms
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
uses: docker/build-push-action@v5
with:
context: .
file: Dockerfile
build-args: |
DFX_NETWORK=local
COMMIT=${{ github.sha }}
cache-from: type=gha,scope=cached-stage
# Exports the artefacts from the final stage
outputs: ./out-mainnet
- name: Get nns-dapp
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
uses: actions/download-artifact@v4
with:
name: nns-dapp
path: out-local
- name: Get sns_aggregator
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
uses: actions/download-artifact@v4
with:
name: sns_aggregator
path: out-local
- name: Get sns_aggregator_dev
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
uses: actions/download-artifact@v4
with:
name: sns_aggregator_dev
path: out-local
- name: Compare wasms
if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true'
run: |
set -x
ls -l
artefacts="sns_aggregator_dev.wasm.gz sns_aggregator.wasm.gz nns-dapp.wasm.gz"
networks=(mainnet local)
for network in "${networks[@]}" ; do
ls -l "out-$network"
(cd "out-$network" && sha256sum ${artefacts[@]} ; ) > "${network}_hashes.txt"
done
diff local_hashes.txt mainnet_hashes.txt || {
echo "ERROR: wasm hashes differ between mainnet and local."
}
aggregator_test:
needs: build
runs-on: ubuntu-20.04
timeout-minutes: 60
steps:
- name: Checkout nns-dapp
uses: actions/checkout@v4
- name: Get sns_aggregator_dev
uses: actions/download-artifact@v4
with:
name: sns_aggregator_dev
- name: Start snapshot environment
uses: ./.github/actions/start_dfx_snapshot
with:
sns_aggregator_wasm: 'sns_aggregator_dev.wasm.gz'
logfile: 'dfx-aggregator-test.log'
- name: Get the earliest data from the sns aggregator
run: |
AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)"
# Wait for the aggregator to be up:
for (( try=300; try>0; try-- )); do
if curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json; then
break
fi
sleep 2
done
expect=10
actual="$(jq length aggregate-1.json)"
# Later we expect 10 SNSs. Make sure that when we do, it's because we
# actually collected the data and it wasn't preloaded from the
# snapshot.
(( actual < expect )) || {
echo ERROR: Should not yet have $expected SNS before collecting.
scripts/sns/aggregator/get_log
}
- name: Verify that configuration is as provided
run: scripts/sns/aggregator/test-config
- name: Make the aggregator collect data quickly
run: dfx canister call sns_aggregator reconfigure '(opt record { update_interval_ms = 100; fast_interval_ms = 1_000_000_000; })'
- name: Wait for the aggregator to get data
run: sleep 120
# sleep time > 12 SNS & 2 block heights each + a few extra calls.
# TODO: The aggregator can be installed and populated in the saved state, so this sleep is not needed.
- name: Get the latest data from the sns aggregator
run: |
AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)"
curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json
expect=10
actual="$(jq length aggregate-1.json)"
(( expect == actual )) || {
echo ERROR: Expected to have $expect SNS in the aggregator but found $actual.
scripts/sns/aggregator/get_log
}
- name: Test the paginated endpoint
run: scripts/sns/aggregator/test-pagination --num 12
- name: Get logs
run: |
scripts/sns/aggregator/get_log > ,logs
LOG_LINES="$(wc -l <,logs)"
(( LOG_LINES > 10 )) || {
echo "ERROR: Expected a non-trivial number of lines to have been logged by now but found only ${LOG_LINES}"
cat ,logs
exit 1
}
- name: Upgrade the aggregator to self with a slow refresh rate
run: dfx canister install --mode upgrade --wasm sns_aggregator_dev.wasm.gz --upgrade-unchanged sns_aggregator '(opt record { update_interval_ms = 1_000_000_000; fast_interval_ms = 1_000_000_000; })' --yes
- name: Expect the paginated data to be retained over the upgrade
run: scripts/sns/aggregator/test-pagination --num 12
- name: Expect the latest data to be retained over the upgrade
run: |
AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)"
curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json
expect=10
actual="$(jq length aggregate-1.json)"
(( expect == actual )) || {
echo ERROR: Expected to have $expect SNS in the aggregator but found $actual.
}
- name: Expect the upstream data to be retained over the upgrade
run: |
./scripts/sns/aggregator/get_stable_data
expect=12
actual="$(jq '.sns_cache.upstream_data | length' stable_data.json)"
(( expect == actual )) || {
echo ERROR: Expected to have $expect SNS in the aggregator upstream data but found $actual.
}
- name: Downgrade sns_aggregator to prod and upgrade back again
run: |
set -euxo pipefail
git fetch --depth 1 origin tag aggregator-prod
diff="$(git diff tags/aggregator-prod rs/sns_aggregator .github/workflows/build.yml)"
if test -n "${diff:-}"
then ./scripts/sns/aggregator/downgrade-upgrade-test -w sns_aggregator_dev.wasm.gz --verbose
else echo "Skipping test as there are no relevant code changes"
fi
- name: Verify that fast data is updated fast
run: |
pushd snsdemo
# Install tools such as quill
bin/dfx-sns-demo-install
# Set canister IDs
bin/dfx-nns-import --network local
popd
scripts/sns/aggregator/test-fast
- name: Stop replica
run: dfx stop
assets:
name: "Upload assets"
needs: build
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: Get docker build outputs
uses: actions/download-artifact@v4
with:
name: out
path: out
- name: Print the hash of all assets
run: find out -type f | xargs sha256sum
- name: 'Record the git commit and any tags'
run: git log | head -n1 > out/commit.txt
- name: 'Upload ${{ matrix.BUILD_NAME }} nns-dapp wasm module'
uses: actions/upload-artifact@v4
with:
name: nns-dapp for ${{ matrix.BUILD_NAME }}
path: |
out/commit.txt
out/nns-dapp.wasm.gz
out/nns-dapp-arg-${{ matrix.DFX_NETWORK }}.did
out/nns-dapp-arg-${{ matrix.DFX_NETWORK }}.bin
out/frontend-config.sh
out/deployment-config.json
- name: 'Upload sns_aggregator wasm module'
uses: actions/upload-artifact@v4
with:
name: sns_aggregator for ${{ matrix.BUILD_NAME }}
path: |
out/sns_aggregator.wasm.gz
out/sns_aggregator_dev.wasm.gz
- name: Release
uses: ./.github/actions/release_nns_dapp
with:
assets_dir: 'out'
token: ${{ secrets.GITHUB_TOKEN }}
- name: 'Upload frontend assets'
uses: actions/upload-artifact@v4
with:
name: NNS frontend assets
path: |
out/assets.tar.xz
out/sourcemaps.tar.xz
- name: "Link the build sha to this commit"
run: |
: Set up git
git config user.name "GitHub Actions Bot"
git config user.email "<>"
: Make a note of the WASM shasum.
NOTE="refs/notes/mainnet/wasm-sha"
SHA="$(sha256sum < "out/nns-dapp.wasm.gz")"
git fetch origin "+${NOTE}:${NOTE}"
if git notes --ref="wasm-sha" add -m "$SHA"
then git push origin "${NOTE}:${NOTE}" || true
else echo SHA already set
fi
- name: "Verify that the WASM module is small enough to deploy"
run: |
wasm_size="$(wc -c < "out/nns-dapp.wasm.gz")"
max_size=3145728
(
echo "## NNS Dapp WASM stats"
humreadable_size="$(numfmt --to=iec-i --suffix=B --format="%.3f" $wasm_size)"
humreadable_max="$(numfmt --to=iec-i --suffix=B --format="%.3f" $max_size )"
humreadable_ratio="$(( (wasm_size * 100) / max_size ))%"
humreadable_free="$(numfmt --to=iec-i --suffix=B --format="%.3f" $(( max_size - wasm_size )))"
echo "**WASM size:** $humreadable_size / $humreadable_max = $humreadable_ratio ($humreadable_free free)"
) | tee -a $GITHUB_STEP_SUMMARY
(( wasm_size <= max_size )) || { echo "The WASM is too large" ; exit 1 ; }
build-pass:
needs: ["build", "test-playwright-e2e-shard-1-of-2", "test-playwright-e2e-shard-2-of-2", "test-rest", "network_independent_wasm", "aggregator_test", "assets", "test-downgrade-upgrade", "test-test-account-api", "test-upgrade-map", "test-upgrade-stable"]
if: ${{ always() }}
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/needs_success
with:
needs: '${{ toJson(needs) }}'