Fix deploy to app (#3768) #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test | |
| # We use `push` events so that we have the actual commit. In `pull_request` | |
| # events we get a merge commit with main instead. The merge commit can be | |
| # useful to check that the code would pass tests once merged, but here it just | |
| # creates confusion and doesn't add anything since the branch must be up to | |
| # date before merge. It's also nice to have CI running on branches without PRs. | |
| on: | |
| push: | |
| workflow_dispatch: | |
| inputs: | |
| no_cache: | |
| description: 'no-cache' | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash -euxlo pipefail {0} | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest-m | |
| timeout-minutes: 45 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Skip build for testing | |
| # Set to true and set a recent `run_id` below to reuse an existing build | |
| # instead of building. | |
| if: false | |
| id: skip_build | |
| run: | | |
| echo "skip_build=true" >> "$GITHUB_OUTPUT" | |
| mkdir out | |
| # The run ID is the number at the end of a URL like this: | |
| # https://github.com/dfinity/nns-dapp/actions/runs/5801187848 | |
| run_id=5801187848 | |
| gh run download "$run_id" --dir ./out -n out | |
| - name: Build nns-dapp repo | |
| if: steps.skip_build.outputs.skip_build != 'true' | |
| uses: ./.github/actions/build_nns_dapp | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: 'Upload nns-dapp wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nns-dapp | |
| path: out/nns-dapp.wasm.gz | |
| retention-days: 3 | |
| - name: 'Upload nns-dapp test wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nns-dapp_test | |
| path: out/nns-dapp_test.wasm.gz | |
| retention-days: 3 | |
| - name: 'Upload sns_aggregator wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sns_aggregator | |
| path: out/sns_aggregator.wasm.gz | |
| retention-days: 3 | |
| - name: 'Upload sns_aggregator_dev wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sns_aggregator_dev | |
| path: out/sns_aggregator_dev.wasm.gz | |
| retention-days: 3 | |
| - name: 'Upload whole out directory' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: out | |
| path: out | |
| retention-days: 3 | |
| test-playwright-e2e-shard-1-of-2: | |
| needs: build | |
| runs-on: ubuntu-latest-m | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Run Playwright e2e test shard 1/2 | |
| uses: ./.github/actions/test-e2e | |
| with: | |
| shard_number: 1 | |
| shard_count: 2 | |
| test-playwright-e2e-shard-2-of-2: | |
| needs: build | |
| runs-on: ubuntu-latest-m | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Run Playwright e2e test shard 2/2 | |
| uses: ./.github/actions/test-e2e | |
| with: | |
| shard_number: 2 | |
| shard_count: 2 | |
| test-downgrade-upgrade: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Get nns-dapp_test | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nns-dapp_test | |
| - name: Start snapshot environment | |
| uses: ./.github/actions/start_dfx_snapshot | |
| with: | |
| nns_dapp_wasm: 'nns-dapp_test.wasm.gz' | |
| - name: Downgrade nns-dapp to prod and upgrade back again | |
| run: ./scripts/nns-dapp/downgrade-upgrade-test -w nns-dapp_test.wasm.gz | |
| - name: Count upgrade cycles | |
| run: scripts/nns-dapp/estimate-upgrade-cycles | tee -a $GITHUB_STEP_SUMMARY | |
| test-test-account-api: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Get nns-dapp_test | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nns-dapp_test | |
| - name: Start empty nns-dapp | |
| # As long as the snapshot environment can be installed with no accounts, we can use that. | |
| # The snapshot action also installs idl2json and jq; commands that we will need. | |
| uses: ./.github/actions/start_dfx_snapshot | |
| with: | |
| nns_dapp_wasm: 'nns-dapp_test.wasm.gz' | |
| - name: Check that test accounts can be created and read | |
| run: ./scripts/nns-dapp/test-account.test | |
| test-rest: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Get nns-dapp | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nns-dapp | |
| - name: Get sns_aggregator | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sns_aggregator | |
| - name: Get sns_aggregator_dev | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sns_aggregator_dev | |
| - name: Start snapshot environment | |
| uses: ./.github/actions/start_dfx_snapshot | |
| with: | |
| nns_dapp_wasm: 'nns-dapp.wasm.gz' | |
| sns_aggregator_wasm: 'sns_aggregator_dev.wasm.gz' | |
| - name: Add go and SNS scripts to the path | |
| run: | | |
| echo "$PWD/snsdemo/bin" >> $GITHUB_PATH | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Install command line HTML parser | |
| run: | | |
| go install github.com/ericchiang/pup@latest | |
| pup --version | |
| - name: Verify that arguments are set in index.html | |
| run: | | |
| for ((i=5; i>0; i--)); do | |
| ( | |
| timeout 60 curl --fail --silent --retry 10 --connect-timeout 5 "http://$(dfx canister id nns-dapp).localhost:8080/" > index.html | |
| file index.html | |
| < index.html gunzip | pup 'head meta[name="nns-dapp-vars"] json{}' | tee nns_dapp_args_in_page.json | |
| ) || { echo "Failed. Retrying..." ; sleep 5 ; continue ; } | |
| break | |
| done | |
| echo "Check a few values:" | |
| for key in data-own-canister-id data-fetch-root-key data-identity-service-url ; do | |
| # Verify that the key is non-trivial: | |
| # `jq -e` returns an error code if the value is missing | |
| # `grep ...` fails if the value is implausibly short. | |
| key="$key" jq -re '.[0][env.key]' nns_dapp_args_in_page.json | grep -E ... | |
| done | |
| - name: Determine ic-wasm version | |
| id: ic-wasm-version | |
| run: | | |
| echo "IC_WASM_VERSION=$(jq -r '.defaults.build.config.IC_WASM_VERSION' dfx.json)" >> "$GITHUB_OUTPUT" | |
| echo "IC_WASM_PATH=/home/runner/.cargo/bin/ic-wasm" >> "$GITHUB_OUTPUT" | |
| - name: Cache ic-wasm | |
| id: cache-ic-wasm | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.ic-wasm-version.outputs.IC_WASM_PATH }} | |
| key: ${{ runner.os }}-${{ steps.ic-wasm-version.outputs.IC_WASM_VERSION }}-ic-wasm | |
| - name: Install cargo binstall | |
| uses: ./.github/actions/install_binstall | |
| - name: Install ic-wasm | |
| if: steps.cache-ic-wasm.outputs.cache-hit != 'true' | |
| run: | | |
| cargo binstall --no-confirm "ic-wasm@${{ steps.ic-wasm-version.outputs.IC_WASM_VERSION }}" | |
| command -v ic-wasm || { | |
| echo "ERROR: Failed to install ic-wasm" | |
| exit 1 | |
| }>&2 | |
| [[ "$( command -v ic-wasm )" == ${{ steps.ic-wasm-version.outputs.IC_WASM_PATH }} ]] || { | |
| echo "ERROR: ic-wasm was installed at an unexpected location." | |
| echo "EXPECTED: ${{ steps.ic-wasm-version.outputs.IC_WASM_PATH }}" | |
| echo "ACTUAL: $( command -v ic-wasm )" | |
| echo "Please align fact and expectation." | |
| exit 1 | |
| } >&2 | |
| - name: Check that metadata is present | |
| run: | | |
| scripts/dfx-wasm-metadata-add.test --verbose | |
| - name: Verify that metrics are present | |
| run: scripts/nns-dapp/e2e-test-metrics-present | |
| - name: Release | |
| run: | | |
| for tag in $(git tag --points-at HEAD) ; do | |
| : Creates or updates a release for the tag | |
| if gh release view "$tag" | |
| then gh release upload --repo dfinity/nns-dapp --clobber "$tag" nns-dapp.wasm.gz || true | |
| else gh release create --title "Release for tags/$tag" --draft --notes "Build artefacts from tag: $tag" "$tag" nns-dapp.wasm.gz | |
| fi | |
| : If the tag is for a proposal or nightly, make it public | |
| [[ "$tag" != proposal-* ]] && [[ "$tag" != nightly-* ]] || { echo "Making release public" ; gh release edit "$tag" --draft=false ; } | |
| done | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Get the postinstall instruction count | |
| run: | | |
| dfx canister install --upgrade-unchanged nns-dapp --wasm nns-dapp.wasm.gz --mode upgrade --argument "$(cat nns-dapp-arg-local.did)" --yes | |
| postinstall_instructions="$(scripts/backend/get_upgrade_instructions)" | |
| echo "Installation consumed ${postinstall_instructions} instructions." | |
| echo "Cycles consumed are instructions * some factor that depends on subnet. There is no guarantee that that formula will not change." | |
| - name: Stop replica | |
| run: dfx stop | |
| network_independent_wasm: | |
| name: "Same wasms for mainnet and local" | |
| # Note: The dockerfile structure SHOULD guarantee that the network is not used in any Wasm build commands. | |
| # As long as that holds, this test is not needed. | |
| needs: build | |
| runs-on: ubuntu-latest-m | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check dockerfile for changes | |
| id: dockerfile_changed | |
| run: | | |
| common_parent_commit="$(git merge-base HEAD origin/main)" | |
| if git diff "$common_parent_commit" Dockerfile | grep -q . | |
| then echo "dockerfile_changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Set up docker buildx | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Create a blank global config | |
| run: echo "{}" > global-config.json | |
| - name: Build wasms | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| build-args: | | |
| DFX_NETWORK=local | |
| COMMIT=${{ github.sha }} | |
| cache-from: type=gha,scope=cached-stage | |
| # Exports the artefacts from the final stage | |
| outputs: ./out-mainnet | |
| - name: Get nns-dapp | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nns-dapp | |
| path: out-local | |
| - name: Get sns_aggregator | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sns_aggregator | |
| path: out-local | |
| - name: Get sns_aggregator_dev | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sns_aggregator_dev | |
| path: out-local | |
| - name: Compare wasms | |
| if: steps.dockerfile_changed.outputs.dockerfile_changed == 'true' | |
| run: | | |
| set -x | |
| ls -l | |
| artefacts="sns_aggregator_dev.wasm.gz sns_aggregator.wasm.gz nns-dapp.wasm.gz" | |
| networks=(mainnet local) | |
| for network in "${networks[@]}" ; do | |
| ls -l "out-$network" | |
| (cd "out-$network" && sha256sum ${artefacts[@]} ; ) > "${network}_hashes.txt" | |
| done | |
| diff local_hashes.txt mainnet_hashes.txt || { | |
| echo "ERROR: wasm hashes differ between mainnet and local." | |
| } | |
| aggregator_test: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout nns-dapp | |
| uses: actions/checkout@v3 | |
| - name: Get sns_aggregator_dev | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sns_aggregator_dev | |
| - name: Start snapshot environment | |
| uses: ./.github/actions/start_dfx_snapshot | |
| with: | |
| sns_aggregator_wasm: 'sns_aggregator_dev.wasm.gz' | |
| - name: Get the earliest data from the sns aggregator | |
| run: | | |
| AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)" | |
| # Wait for the aggregator to be up: | |
| for (( try=300; try>0; try-- )); do | |
| if curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json; then | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| expect=10 | |
| actual="$(jq length aggregate-1.json)" | |
| # Later we expect 10 SNSs. Make sure that when we do, it's because we | |
| # actually collected the data and it wasn't preloaded from the | |
| # snapshot. | |
| (( actual < expect )) || { | |
| echo ERROR: Should not yet have $expected SNS before collecting. | |
| scripts/sns/aggregator/get_log | |
| } | |
| - name: Verify that configuration is as provided | |
| run: scripts/sns/aggregator/test-config | |
| - name: Make the aggregator collect data quickly | |
| run: dfx canister call sns_aggregator reconfigure '(opt record { update_interval_ms = 100; fast_interval_ms = 1_000_000_000; })' | |
| - name: Wait for the aggregator to get data | |
| run: sleep 120 | |
| # sleep time > 12 SNS & 2 block heights each + a few extra calls. | |
| # TODO: The aggregator can be installed and populated in the saved state, so this sleep is not needed. | |
| - name: Get the latest data from the sns aggregator | |
| run: | | |
| AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)" | |
| curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json | |
| expect=10 | |
| actual="$(jq length aggregate-1.json)" | |
| (( expect == actual )) || { | |
| echo ERROR: Expected to have $expect SNS in the aggregator but found $actual. | |
| scripts/sns/aggregator/get_log | |
| } | |
| - name: Test the paginated endpoint | |
| run: scripts/sns/aggregator/test-pagination --num 12 | |
| - name: Get logs | |
| run: | | |
| scripts/sns/aggregator/get_log > ,logs | |
| LOG_LINES="$(wc -l <,logs)" | |
| (( LOG_LINES > 10 )) || { | |
| echo "ERROR: Expected a non-trivial number of lines to have been logged by now but found only ${LOG_LINES}" | |
| cat ,logs | |
| exit 1 | |
| } | |
| - name: Upgrade the aggregator to self with a slow refresh rate | |
| run: dfx canister install --mode upgrade --wasm sns_aggregator_dev.wasm.gz --upgrade-unchanged sns_aggregator '(opt record { update_interval_ms = 1_000_000_000; fast_interval_ms = 1_000_000_000; })' --yes | |
| - name: Expect the paginated data to be retained over the upgrade | |
| run: scripts/sns/aggregator/test-pagination --num 12 | |
| - name: Expect the latest data to be retained over the upgrade | |
| run: | | |
| AGGREGATOR_CANISTER_ID="$(dfx canister id sns_aggregator)" | |
| curl -Lf "http://${AGGREGATOR_CANISTER_ID}.localhost:8080/v1/sns/list/latest/slow.json" | tee aggregate-1.json | |
| expect=10 | |
| actual="$(jq length aggregate-1.json)" | |
| (( expect == actual )) || { | |
| echo ERROR: Expected to have $expect SNS in the aggregator but found $actual. | |
| } | |
| - name: Expect the upstream data to be retained over the upgrade | |
| run: | | |
| ./scripts/sns/aggregator/get_stable_data | |
| expect=12 | |
| actual="$(jq '.sns_cache.upstream_data | length' stable_data.json)" | |
| (( expect == actual )) || { | |
| echo ERROR: Expected to have $expect SNS in the aggregator upstream data but found $actual. | |
| } | |
| - name: Downgrade sns_aggregator to prod and upgrade back again | |
| run: | | |
| set -euxo pipefail | |
| git fetch --depth 1 origin tag aggregator-prod | |
| diff="$(git diff tags/aggregator-prod rs/sns_aggregator .github/workflows/build.yml)" | |
| if test -n "${diff:-}" | |
| then ./scripts/sns/aggregator/downgrade-upgrade-test -w sns_aggregator_dev.wasm.gz --verbose | |
| else echo "Skipping test as there are no relevant code changes" | |
| fi | |
| - name: Verify that fast data is updated fast | |
| run: | | |
| pushd snsdemo | |
| # Install tools such as quill | |
| bin/dfx-sns-demo-install | |
| # Set canister IDs | |
| bin/dfx-nns-import --network local | |
| popd | |
| scripts/sns/aggregator/test-fast | |
| - name: Stop replica | |
| run: dfx stop | |
| assets: | |
| name: "Upload assets" | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Get docker build outputs | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: out | |
| path: out | |
| - name: Print the hash of all assets | |
| run: find out -type f | xargs sha256sum | |
| - name: 'Record the git commit and any tags' | |
| run: git log | head -n1 > out/commit.txt | |
| - name: 'Upload ${{ matrix.BUILD_NAME }} nns-dapp wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: nns-dapp for ${{ matrix.BUILD_NAME }} | |
| path: | | |
| out/commit.txt | |
| out/nns-dapp.wasm.gz | |
| out/nns-dapp-arg-${{ matrix.DFX_NETWORK }}.did | |
| out/nns-dapp-arg-${{ matrix.DFX_NETWORK }}.bin | |
| out/frontend-config.sh | |
| out/deployment-config.json | |
| - name: 'Upload sns_aggregator wasm module' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sns_aggregator for ${{ matrix.BUILD_NAME }} | |
| path: | | |
| out/sns_aggregator.wasm.gz | |
| out/sns_aggregator_dev.wasm.gz | |
| - name: Release | |
| uses: ./.github/actions/release_nns_dapp | |
| with: | |
| assets_dir: 'out' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: 'Upload frontend assets' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: NNS frontend assets | |
| path: | | |
| out/assets.tar.xz | |
| out/sourcemaps.tar.xz | |
| - name: "Link the build sha to this commit" | |
| run: | | |
| : Set up git | |
| git config user.name "GitHub Actions Bot" | |
| git config user.email "<>" | |
| : Make a note of the WASM shasum. | |
| NOTE="refs/notes/mainnet/wasm-sha" | |
| SHA="$(sha256sum < "out/nns-dapp.wasm.gz")" | |
| git fetch origin "+${NOTE}:${NOTE}" | |
| if git notes --ref="wasm-sha" add -m "$SHA" | |
| then git push origin "${NOTE}:${NOTE}" || true | |
| else echo SHA already set | |
| fi | |
| - name: "Verify that the WASM module is small enough to deploy" | |
| run: | | |
| wasm_size="$(wc -c < "out/nns-dapp.wasm.gz")" | |
| max_size=3145728 | |
| ( | |
| echo "## NNS Dapp WASM stats" | |
| humreadable_size="$(numfmt --to=iec-i --suffix=B --format="%.3f" $wasm_size)" | |
| humreadable_max="$(numfmt --to=iec-i --suffix=B --format="%.3f" $max_size )" | |
| humreadable_ratio="$(( (wasm_size * 100) / max_size ))%" | |
| humreadable_free="$(numfmt --to=iec-i --suffix=B --format="%.3f" $(( max_size - wasm_size )))" | |
| echo "**WASM size:** $humreadable_size / $humreadable_max = $humreadable_ratio ($humreadable_free free)" | |
| ) | tee -a $GITHUB_STEP_SUMMARY | |
| (( wasm_size <= max_size )) || { echo "The WASM is too large" ; exit 1 ; } | |
| build-pass: | |
| needs: ["build", "test-playwright-e2e-shard-1-of-2", "test-playwright-e2e-shard-2-of-2", "test-rest", "network_independent_wasm", "aggregator_test", "assets", "test-downgrade-upgrade", "test-test-account-api"] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/needs_success | |
| with: | |
| needs: '${{ toJson(needs) }}' |