Merge pull request #1052 from EmpowerPlastic/upgrade-tests

Upgrade test setup
EmpowerPlastic · Nov 9, 2023 · b8f519c · b8f519c
2 parents 7f01c11 + c0174d6
commit b8f519c
Show file tree

Hide file tree

Showing 6 changed files with 277 additions and 39 deletions.
diff --git a/.github/workflows/chain_genesis-export.yml b/.github/workflows/chain_genesis-export.yml
@@ -2,7 +2,23 @@ name: "Export EmpowerChain Genesis"
 
 on:
   workflow_dispatch:
+    inputs:
+      chain:
+        type: choice
+        description: "Used to get a correct name for the exported genesis file"
+        options:
+          - mainnet
+          - testnet
+      chain_id:
+        description: "Chain ID"
+      state_sync_rpc:
+        description: "State sync rpc"
+      state_sync_peer:
+        description: "State sync peer"
+      current_version:
+        description: "Current version of the binary"
   # TODO: Maybe do on a cron schedule, every week or something?
+  # TODO: Potentially allow for getting some of these values from secrets, especially if we run it as a cron schedule
 
 jobs:
   export-testnet-genesis:
@@ -15,11 +31,11 @@ jobs:
         with:
           version: 0.6.30
 
-      - run: earthly --ci +prep-genesis
+      - run: earthly --ci --no-cache --output +prep-genesis --CHAIN_ID=${{ github.event.inputs.chain_id }}  --STATE_SYNC_RPC=${{ github.event.inputs.state_sync_rpc }} --STATE_SYNC_PEER=${{ github.event.inputs.state_sync_peer }} --CURRENT_VERSION=${{ github.event.inputs.current_version }}
         working-directory: ./chain
 
       - name: Archive genesis file
         uses: actions/upload-artifact@v3
         with:
-          name: exported-testnet-genesis
+          name: exported-${{ github.event.inputs.chain }}-genesis
           path: chain/exported-genesis.json
diff --git a/.github/workflows/chain_upgrade-test.yml b/.github/workflows/chain_upgrade-test.yml
@@ -0,0 +1,36 @@
+name: "Test EmpowerChain Upgrade"
+
+on:
+  workflow_dispatch:
+    inputs:
+      chain:
+        type: choice
+        description: "Used to get a correct name for the exported genesis file"
+        options:
+          - mainnet
+          - testnet
+      current_version:
+        description: "Current version of the binary"
+      upgrade_name:
+        description: "Name of the upgrade"
+  # TODO: Maybe do on a cron schedule, every week or something?
+
+jobs:
+  test-upgrade:
+    name: Test EmpowerChain Upgrade
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: earthly/actions-setup@v1
+        with:
+          version: 0.6.30
+
+      - name: Download a single artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: exported-${{ github.event.inputs.chain }}-genesis
+          path: chain
+
+      - run: earthly --ci --no-cache +upgrade-test --CURRENT_VERSION=${{ github.event.inputs.current_version }} --UPGRADE_NAME=${{ github.event.inputs.upgrade_name }}
+        working-directory: ./chain
diff --git a/chain/Earthfile b/chain/Earthfile
@@ -191,46 +191,45 @@ ibc-conformance-test:
     END
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
-# TODO: Make all the URLs and network version (mainnet/testnet) into ARGS
-prep-genesis:
-    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
-    ARG CHAIN_ID=circulus-1 #empowerchain-1
-    ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
-    ARG STATE_SYNC_PEER=[email protected]:17456 #[email protected]:17456 # TODO: Switch to Empower's own state sync peer
+node-base:
+    ARG --required CURRENT_VERSION
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
         build-essential bsdmainutils git make ncdu gcc \
         git jq chrony liblz4-tool unzip
     RUN wget https://github.com/EmpowerPlastic/empowerchain/releases/download/${CURRENT_VERSION}/empowerd-${CURRENT_VERSION}-linux-amd64.zip
-    RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip
-    RUN ./empowerd init genesis-prepper --chain-id ${CHAIN_ID}
-    RUN echo '#!/bin/bash\n\
-        \n\
-        SNAP_RPC="${STATE_SYNC_RPC}"\n\
-        PEER="${STATE_SYNC_PEER}"\n\
-        \n\
-        LATEST_HEIGHT=$(curl -s $SNAP_RPC/block | jq -r .result.block.header.height)\n\
-        BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000))\n\
-        TRUST_HASH=$(curl -s "$SNAP_RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash)\n\
-        \n\
-        echo SNAP_RPC: $SNAP_RPC\n\
-        echo LATEST_HEIGHT: $LATEST_HEIGHT\n\
-        echo BLOCK_HEIGHT: $BLOCK_HEIGHT\n\
-        echo TRUST_HASH: $TRUST_HASH\n\
-        echo PEER: $PEER\n\
-        \n\
-        sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\\1true| ; \\\n\
-        s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\\1\"$SNAP_RPC,$SNAP_RPC\"| ; \\\n\
-        s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\\1$BLOCK_HEIGHT| ; \\\n\
-        s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\\1\"$TRUST_HASH\"| ; \\\n\
-        s|^(persistent_peers[[:space:]]+=[[:space:]]+).*$|\\1\"$PEER\"|" $HOME/.empowerchain/config/config.toml' > state_sync.sh
-    RUN chmod 700 state_sync.sh
-    RUN ./state_sync.sh
-    RUN ./empowerd start > /dev/null 2>&1 & sleep 5; \
-        while [ "`./empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
-            echo "Waiting to catch up, current block: $(./empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
-            sleep 5; \
-        done
-    RUN ./empowerd export > exported-genesis.json
-    SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
+    RUN mkdir -p /root/binaries
+    RUN unzip -p empowerd-${CURRENT_VERSION}-linux-amd64.zip > /root/binaries/empowerd
+    RUN chmod +x /root/binaries/empowerd
+    ENV PATH $PATH:/root/binaries
+
+# earthly +prep-genesis --CURRENT_VERSION=v1.0.1-rc1 --CHAIN_ID=empowerchain-1 --STATE_SYNC_RPC=https://empower-rpc.polkachu.com:443 --STATE_SYNC_PEER=074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456
+# earthly +prep-genesis --CURRENT_VERSION=v1.0.1-rc1 --CHAIN_ID=circulus-1  --STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 --STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456
+prep-genesis:
+    ARG --required CURRENT_VERSION
+    ARG --required CHAIN_ID
+    ARG --required STATE_SYNC_RPC
+    ARG --required STATE_SYNC_PEER
+    FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    COPY scripts/upgrade-test/state_sync.sh state_sync.sh
+    COPY scripts/upgrade-test/start_and_catch_up.sh start_and_catch_up.sh
+    RUN empowerd init genesis-prepper --chain-id ${CHAIN_ID}
+    RUN ./state_sync.sh ${STATE_SYNC_RPC} ${STATE_SYNC_PEER}
+    RUN ./start_and_catch_up.sh
+    RUN empowerd export > exported-genesis.json
+    SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
+
+upgrade-test:
+    ARG --required CURRENT_VERSION
+    ARG --required UPGRADE_NAME # Need to match the actual upgrade name in app.go. If there is not upgrade or the name is wrong, this will fail
+    FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    COPY scripts/upgrade-test/upgrade.sh upgrade.sh
+    COPY exported-genesis.json exported-genesis.json
+    COPY +build-with-wasmvm/empowerd ./new_empowerd
+    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
+    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
+    ENV LD_LIBRARY_PATH=/usr/local/lib
+
+    RUN ./upgrade.sh ${UPGRADE_NAME}
+
diff --git a/chain/scripts/upgrade-test/start_and_catch_up.sh b/chain/scripts/upgrade-test/start_and_catch_up.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+empowerd start > /tmp/start_and_catch_up 2>&1 &
+sleep 5
+
+max_catch_up_wait_attempts=30; # TODO, MAKE HIGHER
+catch_up_wait_count=0;
+
+while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_wait_count -lt $max_catch_up_wait_attempts ]; do
+    echo "Waiting to catch up, attempt ${catch_up_wait_count}/${max_catch_up_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    catch_up_wait_count=$((catch_up_wait_count+1));
+done
+
+if [ $catch_up_wait_count -eq $max_catch_up_wait_attempts ]; then
+    echo "Failed to wait for catch up after ${max_catch_up_wait_attempts} attempts.";
+    cat /tmp/start_and_catch_up;
+    exit 1;
+fi;
diff --git a/chain/scripts/upgrade-test/state_sync.sh b/chain/scripts/upgrade-test/state_sync.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Good parts of this script is lifted off of the validator Polkachu's typical state_sync setup script
+# Example: https://polkachu.com/state_sync/empower
+
+if [ -z "$1" ]; then
+    echo "Missing STATE_SYNC_RPC"
+    exit 1
+fi
+STATE_SYNC_RPC=$1
+
+if [ -z "$2" ]; then
+    echo "Missing STATE_SYNC_PEER"
+    exit 1
+fi
+STATE_SYNC_PEER=$2
+
+SNAP_RPC="${STATE_SYNC_RPC}"
+PEER="${STATE_SYNC_PEER}"
+
+LATEST_HEIGHT=$(curl -s $SNAP_RPC/block | jq -r .result.block.header.height)
+BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000))
+TRUST_HASH=$(curl -s "$SNAP_RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash)
+
+echo SNAP_RPC: $SNAP_RPC
+echo LATEST_HEIGHT: $LATEST_HEIGHT
+echo BLOCK_HEIGHT: $BLOCK_HEIGHT
+echo TRUST_HASH: $TRUST_HASH
+echo PEER: $PEER
+
+sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \
+s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$SNAP_RPC,$SNAP_RPC\"| ; \
+s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \
+s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"| ; \
+s|^(persistent_peers[[:space:]]+=[[:space:]]+).*$|\1\"$PEER\"|" $HOME/.empowerchain/config/config.toml
diff --git a/chain/scripts/upgrade-test/upgrade.sh b/chain/scripts/upgrade-test/upgrade.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Pre-start, it expects the following:
+# - empowerd binary (in the PATH) of the current version
+# - exported-genesis.json: A raw export from a live chain (testnet or mainnet)
+# - new_empowerd binary (not in PATH, but same directory as this script) of the new version to use for the upgrade
+# This script does the following, roughly:
+# 1: Set up a fresh validator account
+# 2: Manipulate the genesis file to a single-validator genesis (new chain-id and everything) with the new validator account
+# 3: Start empowerd and wait for it to catch up (start up, really)
+# 4: Create and submit a an upgrade proposal
+# 5: Vote for the proposal and wait for the proposal to pass
+# 6: Shut down chain (halted for the upgrade at upgrade height)
+# 7: Start up the new version of empowerd and wait for it to catch up (start up, really)
+
+if [ -z "$1" ]; then
+    echo "Missing first argument UPGRADE_NAME"
+    exit 1
+fi
+UPGRADE_NAME=$1
+
+VALNAME=validator
+VALPASS=passw0rd
+
+# Set up a fresh validator account
+empowerd init ${VALNAME}
+{ echo ${VALPASS}; echo ${VALPASS}; } | empowerd keys add ${VALNAME}
+
+# Set up genesis file as a single-validator genesis (with the new validator account)
+echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis.json ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
+mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
+
+# Lower the voting period to 30s so that the upgrade proposal passes quickly
+sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
+
+empowerd start > /tmp/before_upgrade_log 2>&1 &
+sleep 10;
+
+max_catch_up_attempts=10;
+catch_up_count=0;
+while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_count -lt $max_catch_up_attempts ]; do \
+    echo "Waiting to catch up (attempt ${catch_up_count}/${max_catch_up_attempts}, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    catch_up_count=$((catch_up_count+1));
+done;
+if [ $catch_up_count -eq $max_catch_up_attempts ]; then \
+    echo "Failed to catch up after ${max_catch_up_attempts} attempts.";
+    cat /tmp/before_upgrade_log;
+    exit 1;
+fi;
+sleep 10; # Just for good measure
+
+# Create and submit a an upgrade proposal
+# Set the upgrade height to be current heigh + 20
+UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20));
+echo "Upgrade prop for height $UPGRADE_HEIGHT";
+echo "{
+   \"messages\": [
+    {
+     \"@type\": \"/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade\",
+     \"authority\": \"empower10d07y265gmmuvt4z0w9aw880jnsr700jxwhwvd\",
+     \"plan\": {
+      \"name\": \"${UPGRADE_NAME}\",
+      \"time\": \"0001-01-01T00:00:00Z\",
+      \"height\": \"${UPGRADE_HEIGHT}\",
+      \"info\": \"${UPGRADE_NAME}\",
+      \"upgraded_client_state\": null
+     }
+    }
+   ],
+   \"metadata\": \"ipfs://CID\",
+   \"deposit\": \"2000000000umpwr\",
+   \"title\": \"Test upgrade proposal\",
+   \"summary\": \"Test upgrade proposal summary\"
+          }" > upgrade_proposal.json;
+echo ${VALPASS} | empowerd tx gov submit-proposal upgrade_proposal.json --from validator --yes --chain-id emp-devnet-1;
+sleep 5;
+PROP_ID=$(empowerd q gov proposals --reverse --limit 1 --output json | jq -r ".proposals[0].id");
+
+# Vote and wait for proposal to pass
+echo "Voting on proposal $PROP_ID";
+echo ${VALPASS} | empowerd tx gov vote ${PROP_ID} yes --from validator --yes --chain-id emp-devnet-1;
+sleep 30;
+
+# Double check that the proposal went through
+GOV_RESULT=$(empowerd q gov proposal ${PROP_ID} --output json | jq -r ".status");
+if [ "$GOV_RESULT" != "PROPOSAL_STATUS_PASSED" ]; then \
+  echo "Error: Proposal not passed: $GOV_RESULT";
+  exit 1;
+fi;
+
+# Wait for upgrade height to come around
+max_upgrade_wait_attempts=30;
+upgrade_wait_count=0;
+while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ] && [ $upgrade_wait_count -lt $max_upgrade_wait_attempts ]; do \
+    echo "Waiting for upgrade height (${UPGRADE_HEIGHT}, attempt ${upgrade_wait_count}/${max_upgrade_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    upgrade_wait_count=$((upgrade_wait_count+1));
+done;
+if [ $upgrade_wait_count -eq $max_upgrade_wait_attempts ]; then \
+    echo "Failed to wait for upgrade height ${max_upgrade_wait_attempts} attempts.";
+    cat /tmp/before_upgrade_log;
+    exit 1;
+fi;
+sleep 5; # Just for good measure
+
+# Shut down chain (it is halted at upgrade height, anyway)
+pkill empowerd;
+sleep 5; # Just for good measure
+
+# Start up the new version of empowerd
+./new_empowerd start > /tmp/after_upgrade_log 2>&1 &
+sleep 60; # Let the upgrade take
+
+after_upgrade_max_catch_up_attempts=10;
+after_upgrade_catch_up_count=0;
+while [ "`./new_empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $after_upgrade_catch_up_count -lt $after_upgrade_max_catch_up_attempts ]; do \
+    echo "Waiting to catch up after upgrade (attempt ${after_upgrade_catch_up_count}/${after_upgrade_max_catch_up_attempts}, current block: $(./new_empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    after_upgrade_catch_up_count=$((after_upgrade_catch_up_count+1));
+done;
+if [ $after_upgrade_catch_up_count -eq $after_upgrade_max_catch_up_attempts ]; then \
+    echo "Failed to catch up after the upgrade after ${after_upgrade_max_catch_up_attempts} attempts.";
+    cat /tmp/after_upgrade_log;
+    exit 1;
+fi;
+CURRENT_HEIGHT=$(./new_empowerd status | jq -r ".SyncInfo.latest_block_height");
+if [ $CURRENT_HEIGHT -le $UPGRADE_HEIGHT ]; then \
+    echo "Error: Current height ($CURRENT_HEIGHT) is less than or equal to upgrade height ($UPGRADE_HEIGHT)";
+    cat /tmp/after_upgrade_log;
+    exit 1;
+fi;
+echo "Upgrade test passed! Halted and upgraded on $UPGRADE_HEIGHT, current height: $CURRENT_HEIGHT";