From df40a572e90ae77e3e017dda48c68fdb62dee98c Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Wed, 25 Oct 2023 14:57:40 +1100
Subject: [PATCH 1/8] Add output and remove cache from genesis export GHA

---
 .github/workflows/chain_genesis-export.yml | 2 +-
 chain/Earthfile                            | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/chain_genesis-export.yml b/.github/workflows/chain_genesis-export.yml
index 3fc53bc51..e5118d517 100644
--- a/.github/workflows/chain_genesis-export.yml
+++ b/.github/workflows/chain_genesis-export.yml
@@ -15,7 +15,7 @@ jobs:
         with:
           version: 0.6.30
 
-      - run: earthly --ci +prep-genesis
+      - run: earthly --ci --no-cache --output +prep-genesis
         working-directory: ./chain
 
       - name: Archive genesis file
diff --git a/chain/Earthfile b/chain/Earthfile
index a9d7a9139..80cac7a5e 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -191,7 +191,6 @@ ibc-conformance-test:
     END
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
-# TODO: Make all the URLs and network version (mainnet/testnet) into ARGS
 prep-genesis:
     ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG CHAIN_ID=circulus-1 #empowerchain-1

From 2516ee4c3ad7fa4851dbcbf73a09cf0cce86b932 Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Wed, 25 Oct 2023 18:58:00 +1100
Subject: [PATCH 2/8] Set up single validator from genesis file

---
 chain/Earthfile | 45 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/chain/Earthfile b/chain/Earthfile
index 80cac7a5e..012785b1e 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -191,19 +191,24 @@ ibc-conformance-test:
     END
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
-prep-genesis:
+node-base:
     ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
-    ARG CHAIN_ID=circulus-1 #empowerchain-1
-    ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
-    ARG STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456 #074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456 # TODO: Switch to Empower's own state sync peer
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
         build-essential bsdmainutils git make ncdu gcc \
         git jq chrony liblz4-tool unzip
     RUN wget https://github.com/EmpowerPlastic/empowerchain/releases/download/${CURRENT_VERSION}/empowerd-${CURRENT_VERSION}-linux-amd64.zip
-    RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip
-    RUN ./empowerd init genesis-prepper --chain-id ${CHAIN_ID}
+    RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip -d /root/binaries
+    ENV PATH $PATH:/root/binaries
+
+prep-genesis:
+    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CHAIN_ID=circulus-1 #empowerchain-1
+    ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
+    ARG STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456 #074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456 # TODO: Switch to Empower's own state sync peer
+    FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    RUN empowerd init genesis-prepper --chain-id ${CHAIN_ID}
     RUN echo '#!/bin/bash\n\
         \n\
         SNAP_RPC="${STATE_SYNC_RPC}"\n\
@@ -226,10 +231,28 @@ prep-genesis:
         s|^(persistent_peers[[:space:]]+=[[:space:]]+).*$|\\1\"$PEER\"|" $HOME/.empowerchain/config/config.toml' > state_sync.sh
     RUN chmod 700 state_sync.sh
     RUN ./state_sync.sh
-    RUN ./empowerd start > /dev/null 2>&1 & sleep 5; \
-        while [ "`./empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
-            echo "Waiting to catch up, current block: $(./empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+    RUN empowerd start > /dev/null 2>&1 & sleep 5; \
+        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
+            echo "Waiting to catch up, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
             sleep 5; \
         done
-    RUN ./empowerd export > exported-genesis.json
-    SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
\ No newline at end of file
+    RUN empowerd export > exported-genesis.json
+    SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
+
+upgrade-test:
+    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG VALPASS=passw0rd
+    ARG VALNAME=validator
+    ARG --required UPGRADE_NAME # TODO: Could this be parsed from CURRENT_VERSION if we keep a sane naming scheme?
+    FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    COPY exported-genesis.json exported-genesis.json
+
+    RUN empowerd init ${VALNAME}
+    # Set up validator key
+    RUN { echo ${VALPASS}; echo ${VALPASS}; } | empowerd keys add ${VALNAME}
+
+        # Set up devnet genesis for new key
+    RUN echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
+    RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
+
+    RUN false
\ No newline at end of file

From 2f642549b1271e1f28b3aa2b64560bfdd3fecbfe Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Fri, 27 Oct 2023 19:22:21 +1100
Subject: [PATCH 3/8] Use v1.0.1-rc0 for upgrade ttest

---
 chain/Earthfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/chain/Earthfile b/chain/Earthfile
index 012785b1e..b18163d83 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -192,7 +192,7 @@ ibc-conformance-test:
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
 node-base:
-    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
@@ -203,7 +203,7 @@ node-base:
     ENV PATH $PATH:/root/binaries
 
 prep-genesis:
-    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG CHAIN_ID=circulus-1 #empowerchain-1
     ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
     ARG STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456 #074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456 # TODO: Switch to Empower's own state sync peer
@@ -240,7 +240,7 @@ prep-genesis:
     SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
 
 upgrade-test:
-    ARG CURRENT_VERSION=v1.0.0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG VALPASS=passw0rd
     ARG VALNAME=validator
     ARG --required UPGRADE_NAME # TODO: Could this be parsed from CURRENT_VERSION if we keep a sane naming scheme?
@@ -252,7 +252,7 @@ upgrade-test:
     RUN { echo ${VALPASS}; echo ${VALPASS}; } | empowerd keys add ${VALNAME}
 
         # Set up devnet genesis for new key
-    RUN echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
+    RUN echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis.json ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
     RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
 
     RUN false
\ No newline at end of file

From e24222b115aaf69ec9c8bd433eb56b8b418d7812 Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Fri, 27 Oct 2023 19:35:26 +1100
Subject: [PATCH 4/8] Fix test failing for non-existen dist module account

---
 chain/app/genesis-tools/genesis-single-validator.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/chain/app/genesis-tools/genesis-single-validator.go b/chain/app/genesis-tools/genesis-single-validator.go
index 833943f8c..559bf225b 100644
--- a/chain/app/genesis-tools/genesis-single-validator.go
+++ b/chain/app/genesis-tools/genesis-single-validator.go
@@ -117,14 +117,13 @@ func SetDistributionSingleValidator(cdc codec.Codec, genesisState *GenesisState,
 
 	// Get distribution module and set community pool == distribution module balance (rewards are now removed, so this is to get the balance to add up correctly)
 	distModuleAccount, found := GetModuleAccount(cdc, genesisState, distrtypes.ModuleName)
-	if !found {
-		log.Panicf("%s module account not found", distrtypes.ModuleName)
-	}
-	distrModuleBalance, found := GetBalanceOfAddress(genesisState, distModuleAccount.GetAddress().String(), params.BaseCoinDenom)
-	if !found {
-		log.Panicf("%s distrModuleBalance not found", distrtypes.ModuleName)
+	if found {
+		distrModuleBalance, found := GetBalanceOfAddress(genesisState, distModuleAccount.GetAddress().String(), params.BaseCoinDenom)
+		if !found {
+			log.Panicf("%s distrModuleBalance not found", distrtypes.ModuleName)
+		}
+		genesisState.DistrGenesis.FeePool.CommunityPool = sdk.NewDecCoins(sdk.NewDecCoin(params.BaseCoinDenom, distrModuleBalance.Amount))
 	}
-	genesisState.DistrGenesis.FeePool.CommunityPool = sdk.NewDecCoins(sdk.NewDecCoin(params.BaseCoinDenom, distrModuleBalance.Amount))
 
 	// Since we are removing all previous validators, we need to remove the slashing events to make event numbers add up correctly
 	genesisState.DistrGenesis.ValidatorSlashEvents = []distrtypes.ValidatorSlashEventRecord{}

From b8e57d6c604b0a2be1ba0a30f82fa21f8512fb18 Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Mon, 30 Oct 2023 18:27:26 +1100
Subject: [PATCH 5/8] Add upgrade prop to upgrade test script

---
 chain/Earthfile | 51 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/chain/Earthfile b/chain/Earthfile
index b18163d83..f3fa7b3c7 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -254,5 +254,56 @@ upgrade-test:
         # Set up devnet genesis for new key
     RUN echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis.json ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
     RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
+    RUN sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
+
+    RUN empowerd start > /dev/null 2>&1 & sleep 10; \
+        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
+            echo "Waiting to catch up, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+            sleep 5; \
+        done; \
+        sleep 10; \
+        UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20)); \
+        echo "Upgrade prop for height $UPGRADE_HEIGHT"; \
+        echo "{\n\
+           \"messages\": [\n\
+            {\n\
+             \"@type\": \"/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade\",\n\
+             \"authority\": \"empower10d07y265gmmuvt4z0w9aw880jnsr700jxwhwvd\",\n\
+             \"plan\": {\n\
+              \"name\": \"${UPGRADE_NAME}\",\n\
+              \"time\": \"0001-01-01T00:00:00Z\",\n\
+              \"height\": \"${UPGRADE_HEIGHT}\",\n\
+              \"info\": \"${UPGRADE_NAME}\",\n\
+              \"upgraded_client_state\": null\n\
+             }\n\
+            }\n\
+           ],\n\
+           \"metadata\": \"ipfs://CID\",\n\
+           \"deposit\": \"2000000000umpwr\",\n\
+           \"title\": \"Test upgrade proposal\",\n\
+           \"summary\": \"Test upgrade proposal summary\"\n\
+                  }" > upgrade_proposal.json; \
+        echo ${VALPASS} | empowerd tx gov submit-proposal upgrade_proposal.json --from validator --yes --chain-id emp-devnet-1; \
+        sleep 5; \
+        PROP_ID=$(empowerd q gov proposals --reverse --limit 1 --output json | jq -r ".proposals[0].id"); \
+        echo "Voting on proposal $PROP_ID"; \
+        echo ${VALPASS} | empowerd tx gov vote ${PROP_ID} yes --from validator --yes --chain-id emp-devnet-1; \
+        sleep 30; \
+        GOV_RESULT=$(empowerd q gov proposal ${PROP_ID} --output json | jq -r ".status"); \
+        if [ "$GOV_RESULT" != "PROPOSAL_STATUS_PASSED" ]; then \
+          echo "Error: Proposal not passed: $GOV_RESULT"; \
+          exit 1; \
+        fi; \
+        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ]; do \
+            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+            sleep 5; \
+        done; \
+
+    COPY +build-with-wasmvm/empowerd ./new_empowerd
+    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
+    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
+    ENV LD_LIBRARY_PATH=/usr/local/lib
+
+    # TODO: Next up: start with the new binary and let it run for a few blocks
 
     RUN false
\ No newline at end of file

From be6c52e988252cfef70bc7a50cc4bec9990d8977 Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Tue, 31 Oct 2023 17:32:49 +1100
Subject: [PATCH 6/8] Full upgrade test

---
 .github/workflows/chain_upgrade-test.yml | 25 +++++++++
 chain/Earthfile                          | 71 ++++++++++++++++++------
 2 files changed, 78 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/chain_upgrade-test.yml

diff --git a/.github/workflows/chain_upgrade-test.yml b/.github/workflows/chain_upgrade-test.yml
new file mode 100644
index 000000000..5b8294fa4
--- /dev/null
+++ b/.github/workflows/chain_upgrade-test.yml
@@ -0,0 +1,25 @@
+name: "Test EmpowerChain Upgrade"
+
+on:
+  workflow_dispatch:
+  # TODO: Maybe do on a cron schedule, every week or something?
+
+jobs:
+  test-testnet-upgrade:
+    name: Test EmpowerChain Testnet Upgrade
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: earthly/actions-setup@v1
+        with:
+          version: 0.6.30
+
+      - name: Download a single artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: exported-testnet-genesis
+          path: chain
+
+      - run: earthly --ci --no-cache +upgrade-test --UPGRADE_NAME=v2 # Rely on defaults for other args/env vars
+        working-directory: ./chain
\ No newline at end of file
diff --git a/chain/Earthfile b/chain/Earthfile
index 48de36d91..f22a9d1c4 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -192,7 +192,7 @@ ibc-conformance-test:
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
 node-base:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
@@ -202,8 +202,9 @@ node-base:
     RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip -d /root/binaries
     ENV PATH $PATH:/root/binaries
 
+# earthly +prep-genesis --CHAIN_ID=empowerchain-1 --STATE_SYNC_RPC=https://empower-rpc.polkachu.com:443 --STATE_SYNC_PEER=074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456
 prep-genesis:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG CHAIN_ID=circulus-1 #empowerchain-1
     ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
     ARG STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456 #074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456 # TODO: Switch to Empower's own state sync peer
@@ -240,12 +241,16 @@ prep-genesis:
     SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
 
 upgrade-test:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG VALPASS=passw0rd
     ARG VALNAME=validator
     ARG --required UPGRADE_NAME # TODO: Could this be parsed from CURRENT_VERSION if we keep a sane naming scheme?
     FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
     COPY exported-genesis.json exported-genesis.json
+    COPY +build-with-wasmvm/empowerd ./new_empowerd
+    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
+    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
+    ENV LD_LIBRARY_PATH=/usr/local/lib
 
     RUN empowerd init ${VALNAME}
     # Set up validator key
@@ -256,12 +261,20 @@ upgrade-test:
     RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
     RUN sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
 
-    RUN empowerd start > /dev/null 2>&1 & sleep 10; \
-        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
-            echo "Waiting to catch up, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+    RUN empowerd start > /tmp/before_upgrade_log 2>&1 & sleep 10; \
+        max_catch_up_attempts=10; \
+        catch_up_count=0; \
+        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_count -lt $max_catch_up_attempts ]; do \
+            echo "Waiting to catch up (attempt ${catch_up_count}/${max_catch_up_attempts}, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
             sleep 5; \
+            catch_up_count=$((catch_up_count+1)); \
         done; \
-        sleep 10; \
+        if [ $catch_up_count -eq $max_catch_up_attempts ]; then \
+            echo "Failed to catch up after ${max_catch_up_attempts} attempts."; \
+            cat /tmp/before_upgrade_log; \
+            exit 1; \
+        fi; \
+        sleep 10; \ # Just for good measure
         UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20)); \
         echo "Upgrade prop for height $UPGRADE_HEIGHT"; \
         echo "{\n\
@@ -294,16 +307,38 @@ upgrade-test:
           echo "Error: Proposal not passed: $GOV_RESULT"; \
           exit 1; \
         fi; \
-        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ]; do \
-            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+        max_upgrade_wait_attempts=30; \
+        upgrade_wait_count=0; \
+        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ] && [ $upgrade_wait_count -lt $max_upgrade_wait_attempts ]; do \
+            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}, attempt ${upgrade_wait_count}/${max_upgrade_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
             sleep 5; \
+            upgrade_wait_count=$((upgrade_wait_count+1)); \
         done; \
-
-    COPY +build-with-wasmvm/empowerd ./new_empowerd
-    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
-    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
-    ENV LD_LIBRARY_PATH=/usr/local/lib
-
-    # TODO: Next up: start with the new binary and let it run for a few blocks
-
-    RUN false
\ No newline at end of file
+        if [ $upgrade_wait_count -eq $max_upgrade_wait_attempts ]; then \
+            echo "Failed to wait for upgrade height ${max_upgrade_wait_attempts} attempts."; \
+            cat /tmp/before_upgrade_log; \
+            exit 1; \
+        fi; \
+        sleep 5; \ # Just for good measure
+        pkill empowerd; \
+        sleep 5; \ # Just for good measure
+        ./new_empowerd start > /tmp/after_upgrade_log 2>&1 & sleep 60; \
+        after_upgrade_max_catch_up_attempts=10; \
+        after_upgrade_catch_up_count=0; \
+        while [ "`./new_empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $after_upgrade_catch_up_count -lt $after_upgrade_max_catch_up_attempts ]; do \
+            echo "Waiting to catch up after upgrade (attempt ${after_upgrade_catch_up_count}/${after_upgrade_max_catch_up_attempts}, current block: $(./new_empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+            sleep 5; \
+            after_upgrade_catch_up_count=$((after_upgrade_catch_up_count+1)); \
+        done; \
+        if [ $after_upgrade_catch_up_count -eq $after_upgrade_max_catch_up_attempts ]; then \
+            echo "Failed to catch up after the upgrade after ${after_upgrade_max_catch_up_attempts} attempts."; \
+            cat /tmp/after_upgrade_log; \
+            exit 1; \
+        fi; \
+        CURRENT_HEIGHT=$(./new_empowerd status | jq -r ".SyncInfo.latest_block_height"); \
+        if [ $CURRENT_HEIGHT -le $UPGRADE_HEIGHT ]; then \
+            echo "Error: Current height ($CURRENT_HEIGHT) is less than or equal to upgrade height ($UPGRADE_HEIGHT)"; \
+            cat /tmp/after_upgrade_log; \
+            exit 1; \
+        fi; \
+        echo "Upgrade test passed! Halted and upgraded on $UPGRADE_HEIGHT, current height: $CURRENT_HEIGHT";
\ No newline at end of file

From cf0dc2b6772025d9c6ddb532ca85407a13eab678 Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Tue, 7 Nov 2023 14:39:28 +1100
Subject: [PATCH 7/8] Move upgrade scrits into external bash script files

---
 chain/Earthfile                               | 145 +++---------------
 .../upgrade-test/start_and_catch_up.sh        |  19 +++
 chain/scripts/upgrade-test/state_sync.sh      |  35 +++++
 chain/scripts/upgrade-test/upgrade.sh         | 133 ++++++++++++++++
 4 files changed, 205 insertions(+), 127 deletions(-)
 create mode 100755 chain/scripts/upgrade-test/start_and_catch_up.sh
 create mode 100755 chain/scripts/upgrade-test/state_sync.sh
 create mode 100755 chain/scripts/upgrade-test/upgrade.sh

diff --git a/chain/Earthfile b/chain/Earthfile
index f22a9d1c4..cc97ae801 100644
--- a/chain/Earthfile
+++ b/chain/Earthfile
@@ -192,153 +192,44 @@ ibc-conformance-test:
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
 node-base:
-    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG --required CURRENT_VERSION
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
         build-essential bsdmainutils git make ncdu gcc \
         git jq chrony liblz4-tool unzip
     RUN wget https://github.com/EmpowerPlastic/empowerchain/releases/download/${CURRENT_VERSION}/empowerd-${CURRENT_VERSION}-linux-amd64.zip
-    RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip -d /root/binaries
+    RUN mkdir -p /root/binaries
+    RUN unzip -p empowerd-${CURRENT_VERSION}-linux-amd64.zip > /root/binaries/empowerd
+    RUN chmod +x /root/binaries/empowerd
     ENV PATH $PATH:/root/binaries
 
-# earthly +prep-genesis --CHAIN_ID=empowerchain-1 --STATE_SYNC_RPC=https://empower-rpc.polkachu.com:443 --STATE_SYNC_PEER=074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456
+# earthly +prep-genesis --CURRENT_VERSION=v1.0.1-rc1 --CHAIN_ID=empowerchain-1 --STATE_SYNC_RPC=https://empower-rpc.polkachu.com:443 --STATE_SYNC_PEER=074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456
+# earthly +prep-genesis --CURRENT_VERSION=v1.0.1-rc1 --CHAIN_ID=circulus-1  --STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 --STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456
 prep-genesis:
-    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
-    ARG CHAIN_ID=circulus-1 #empowerchain-1
-    ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
-    ARG STATE_SYNC_PEER=95ea7999e3ecd3fb7fd73fae70b3b29a6af24c8d@46.4.5.45:17456 #074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456 # TODO: Switch to Empower's own state sync peer
+    ARG --required CURRENT_VERSION
+    ARG --required CHAIN_ID
+    ARG --required STATE_SYNC_RPC
+    ARG --required STATE_SYNC_PEER
     FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    COPY scripts/upgrade-test/state_sync.sh state_sync.sh
+    COPY scripts/upgrade-test/start_and_catch_up.sh start_and_catch_up.sh
     RUN empowerd init genesis-prepper --chain-id ${CHAIN_ID}
-    RUN echo '#!/bin/bash\n\
-        \n\
-        SNAP_RPC="${STATE_SYNC_RPC}"\n\
-        PEER="${STATE_SYNC_PEER}"\n\
-        \n\
-        LATEST_HEIGHT=$(curl -s $SNAP_RPC/block | jq -r .result.block.header.height)\n\
-        BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000))\n\
-        TRUST_HASH=$(curl -s "$SNAP_RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash)\n\
-        \n\
-        echo SNAP_RPC: $SNAP_RPC\n\
-        echo LATEST_HEIGHT: $LATEST_HEIGHT\n\
-        echo BLOCK_HEIGHT: $BLOCK_HEIGHT\n\
-        echo TRUST_HASH: $TRUST_HASH\n\
-        echo PEER: $PEER\n\
-        \n\
-        sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\\1true| ; \\\n\
-        s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\\1\"$SNAP_RPC,$SNAP_RPC\"| ; \\\n\
-        s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\\1$BLOCK_HEIGHT| ; \\\n\
-        s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\\1\"$TRUST_HASH\"| ; \\\n\
-        s|^(persistent_peers[[:space:]]+=[[:space:]]+).*$|\\1\"$PEER\"|" $HOME/.empowerchain/config/config.toml' > state_sync.sh
-    RUN chmod 700 state_sync.sh
-    RUN ./state_sync.sh
-    RUN empowerd start > /dev/null 2>&1 & sleep 5; \
-        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
-            echo "Waiting to catch up, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
-            sleep 5; \
-        done
+    RUN ./state_sync.sh ${STATE_SYNC_RPC} ${STATE_SYNC_PEER}
+    RUN ./start_and_catch_up.sh
     RUN empowerd export > exported-genesis.json
     SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
 
 upgrade-test:
-    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
-    ARG VALPASS=passw0rd
-    ARG VALNAME=validator
-    ARG --required UPGRADE_NAME # TODO: Could this be parsed from CURRENT_VERSION if we keep a sane naming scheme?
+    ARG --required CURRENT_VERSION
+    ARG --required UPGRADE_NAME # Need to match the actual upgrade name in app.go. If there is not upgrade or the name is wrong, this will fail
     FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
+    COPY scripts/upgrade-test/upgrade.sh upgrade.sh
     COPY exported-genesis.json exported-genesis.json
     COPY +build-with-wasmvm/empowerd ./new_empowerd
     COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
     COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
     ENV LD_LIBRARY_PATH=/usr/local/lib
 
-    RUN empowerd init ${VALNAME}
-    # Set up validator key
-    RUN { echo ${VALPASS}; echo ${VALPASS}; } | empowerd keys add ${VALNAME}
+    RUN ./upgrade.sh ${UPGRADE_NAME}
 
-        # Set up devnet genesis for new key
-    RUN echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis.json ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
-    RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
-    RUN sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
-
-    RUN empowerd start > /tmp/before_upgrade_log 2>&1 & sleep 10; \
-        max_catch_up_attempts=10; \
-        catch_up_count=0; \
-        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_count -lt $max_catch_up_attempts ]; do \
-            echo "Waiting to catch up (attempt ${catch_up_count}/${max_catch_up_attempts}, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
-            sleep 5; \
-            catch_up_count=$((catch_up_count+1)); \
-        done; \
-        if [ $catch_up_count -eq $max_catch_up_attempts ]; then \
-            echo "Failed to catch up after ${max_catch_up_attempts} attempts."; \
-            cat /tmp/before_upgrade_log; \
-            exit 1; \
-        fi; \
-        sleep 10; \ # Just for good measure
-        UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20)); \
-        echo "Upgrade prop for height $UPGRADE_HEIGHT"; \
-        echo "{\n\
-           \"messages\": [\n\
-            {\n\
-             \"@type\": \"/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade\",\n\
-             \"authority\": \"empower10d07y265gmmuvt4z0w9aw880jnsr700jxwhwvd\",\n\
-             \"plan\": {\n\
-              \"name\": \"${UPGRADE_NAME}\",\n\
-              \"time\": \"0001-01-01T00:00:00Z\",\n\
-              \"height\": \"${UPGRADE_HEIGHT}\",\n\
-              \"info\": \"${UPGRADE_NAME}\",\n\
-              \"upgraded_client_state\": null\n\
-             }\n\
-            }\n\
-           ],\n\
-           \"metadata\": \"ipfs://CID\",\n\
-           \"deposit\": \"2000000000umpwr\",\n\
-           \"title\": \"Test upgrade proposal\",\n\
-           \"summary\": \"Test upgrade proposal summary\"\n\
-                  }" > upgrade_proposal.json; \
-        echo ${VALPASS} | empowerd tx gov submit-proposal upgrade_proposal.json --from validator --yes --chain-id emp-devnet-1; \
-        sleep 5; \
-        PROP_ID=$(empowerd q gov proposals --reverse --limit 1 --output json | jq -r ".proposals[0].id"); \
-        echo "Voting on proposal $PROP_ID"; \
-        echo ${VALPASS} | empowerd tx gov vote ${PROP_ID} yes --from validator --yes --chain-id emp-devnet-1; \
-        sleep 30; \
-        GOV_RESULT=$(empowerd q gov proposal ${PROP_ID} --output json | jq -r ".status"); \
-        if [ "$GOV_RESULT" != "PROPOSAL_STATUS_PASSED" ]; then \
-          echo "Error: Proposal not passed: $GOV_RESULT"; \
-          exit 1; \
-        fi; \
-        max_upgrade_wait_attempts=30; \
-        upgrade_wait_count=0; \
-        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ] && [ $upgrade_wait_count -lt $max_upgrade_wait_attempts ]; do \
-            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}, attempt ${upgrade_wait_count}/${max_upgrade_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
-            sleep 5; \
-            upgrade_wait_count=$((upgrade_wait_count+1)); \
-        done; \
-        if [ $upgrade_wait_count -eq $max_upgrade_wait_attempts ]; then \
-            echo "Failed to wait for upgrade height ${max_upgrade_wait_attempts} attempts."; \
-            cat /tmp/before_upgrade_log; \
-            exit 1; \
-        fi; \
-        sleep 5; \ # Just for good measure
-        pkill empowerd; \
-        sleep 5; \ # Just for good measure
-        ./new_empowerd start > /tmp/after_upgrade_log 2>&1 & sleep 60; \
-        after_upgrade_max_catch_up_attempts=10; \
-        after_upgrade_catch_up_count=0; \
-        while [ "`./new_empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $after_upgrade_catch_up_count -lt $after_upgrade_max_catch_up_attempts ]; do \
-            echo "Waiting to catch up after upgrade (attempt ${after_upgrade_catch_up_count}/${after_upgrade_max_catch_up_attempts}, current block: $(./new_empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
-            sleep 5; \
-            after_upgrade_catch_up_count=$((after_upgrade_catch_up_count+1)); \
-        done; \
-        if [ $after_upgrade_catch_up_count -eq $after_upgrade_max_catch_up_attempts ]; then \
-            echo "Failed to catch up after the upgrade after ${after_upgrade_max_catch_up_attempts} attempts."; \
-            cat /tmp/after_upgrade_log; \
-            exit 1; \
-        fi; \
-        CURRENT_HEIGHT=$(./new_empowerd status | jq -r ".SyncInfo.latest_block_height"); \
-        if [ $CURRENT_HEIGHT -le $UPGRADE_HEIGHT ]; then \
-            echo "Error: Current height ($CURRENT_HEIGHT) is less than or equal to upgrade height ($UPGRADE_HEIGHT)"; \
-            cat /tmp/after_upgrade_log; \
-            exit 1; \
-        fi; \
-        echo "Upgrade test passed! Halted and upgraded on $UPGRADE_HEIGHT, current height: $CURRENT_HEIGHT";
\ No newline at end of file
diff --git a/chain/scripts/upgrade-test/start_and_catch_up.sh b/chain/scripts/upgrade-test/start_and_catch_up.sh
new file mode 100755
index 000000000..ae1243161
--- /dev/null
+++ b/chain/scripts/upgrade-test/start_and_catch_up.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+empowerd start > /tmp/start_and_catch_up 2>&1 &
+sleep 5
+
+max_catch_up_wait_attempts=30; # TODO, MAKE HIGHER
+catch_up_wait_count=0;
+
+while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_wait_count -lt $max_catch_up_wait_attempts ]; do
+    echo "Waiting to catch up, attempt ${catch_up_wait_count}/${max_catch_up_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    catch_up_wait_count=$((catch_up_wait_count+1));
+done
+
+if [ $catch_up_wait_count -eq $max_catch_up_wait_attempts ]; then
+    echo "Failed to wait for catch up after ${max_catch_up_wait_attempts} attempts.";
+    cat /tmp/start_and_catch_up;
+    exit 1;
+fi;
\ No newline at end of file
diff --git a/chain/scripts/upgrade-test/state_sync.sh b/chain/scripts/upgrade-test/state_sync.sh
new file mode 100755
index 000000000..4653153d9
--- /dev/null
+++ b/chain/scripts/upgrade-test/state_sync.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Good parts of this script is lifted off of the validator Polkachu's typical state_sync setup script
+# Example: https://polkachu.com/state_sync/empower
+
+if [ -z "$1" ]; then
+    echo "Missing STATE_SYNC_RPC"
+    exit 1
+fi
+STATE_SYNC_RPC=$1
+
+if [ -z "$2" ]; then
+    echo "Missing STATE_SYNC_PEER"
+    exit 1
+fi
+STATE_SYNC_PEER=$2
+
+SNAP_RPC="${STATE_SYNC_RPC}"
+PEER="${STATE_SYNC_PEER}"
+
+LATEST_HEIGHT=$(curl -s $SNAP_RPC/block | jq -r .result.block.header.height)
+BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000))
+TRUST_HASH=$(curl -s "$SNAP_RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash)
+
+echo SNAP_RPC: $SNAP_RPC
+echo LATEST_HEIGHT: $LATEST_HEIGHT
+echo BLOCK_HEIGHT: $BLOCK_HEIGHT
+echo TRUST_HASH: $TRUST_HASH
+echo PEER: $PEER
+
+sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \
+s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$SNAP_RPC,$SNAP_RPC\"| ; \
+s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \
+s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"| ; \
+s|^(persistent_peers[[:space:]]+=[[:space:]]+).*$|\1\"$PEER\"|" $HOME/.empowerchain/config/config.toml
\ No newline at end of file
diff --git a/chain/scripts/upgrade-test/upgrade.sh b/chain/scripts/upgrade-test/upgrade.sh
new file mode 100755
index 000000000..5f3fbc10f
--- /dev/null
+++ b/chain/scripts/upgrade-test/upgrade.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Pre-start, it expects the following:
+# - empowerd binary (in the PATH) of the current version
+# - exported-genesis.json: A raw export from a live chain (testnet or mainnet)
+# - new_empowerd binary (not in PATH, but same directory as this script) of the new version to use for the upgrade
+# This script does the following, roughly:
+# 1: Set up a fresh validator account
+# 2: Manipulate the genesis file to a single-validator genesis (new chain-id and everything) with the new validator account
+# 3: Start empowerd and wait for it to catch up (start up, really)
+# 4: Create and submit a an upgrade proposal
+# 5: Vote for the proposal and wait for the proposal to pass
+# 6: Shut down chain (halted for the upgrade at upgrade height)
+# 7: Start up the new version of empowerd and wait for it to catch up (start up, really)
+
+if [ -z "$1" ]; then
+    echo "Missing first argument UPGRADE_NAME"
+    exit 1
+fi
+UPGRADE_NAME=$1
+
+VALNAME=validator
+VALPASS=passw0rd
+
+# Set up a fresh validator account
+empowerd init ${VALNAME}
+{ echo ${VALPASS}; echo ${VALPASS}; } | empowerd keys add ${VALNAME}
+
+# Set up genesis file as a single-validator genesis (with the new validator account)
+echo ${VALPASS} | empowerd genesis single-validator ./exported-genesis.json ${VALNAME} $(jq -r .pub_key.value ~/.empowerchain/config/priv_validator_key.json)
+mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
+
+# Lower the voting period to 30s so that the upgrade proposal passes quickly
+sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
+
+empowerd start > /tmp/before_upgrade_log 2>&1 &
+sleep 10;
+
+max_catch_up_attempts=10;
+catch_up_count=0;
+while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_count -lt $max_catch_up_attempts ]; do \
+    echo "Waiting to catch up (attempt ${catch_up_count}/${max_catch_up_attempts}, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    catch_up_count=$((catch_up_count+1));
+done;
+if [ $catch_up_count -eq $max_catch_up_attempts ]; then \
+    echo "Failed to catch up after ${max_catch_up_attempts} attempts.";
+    cat /tmp/before_upgrade_log;
+    exit 1;
+fi;
+sleep 10; # Just for good measure
+
+# Create and submit a an upgrade proposal
+# Set the upgrade height to be current heigh + 20
+UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20));
+echo "Upgrade prop for height $UPGRADE_HEIGHT";
+echo "{
+   \"messages\": [
+    {
+     \"@type\": \"/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade\",
+     \"authority\": \"empower10d07y265gmmuvt4z0w9aw880jnsr700jxwhwvd\",
+     \"plan\": {
+      \"name\": \"${UPGRADE_NAME}\",
+      \"time\": \"0001-01-01T00:00:00Z\",
+      \"height\": \"${UPGRADE_HEIGHT}\",
+      \"info\": \"${UPGRADE_NAME}\",
+      \"upgraded_client_state\": null
+     }
+    }
+   ],
+   \"metadata\": \"ipfs://CID\",
+   \"deposit\": \"2000000000umpwr\",
+   \"title\": \"Test upgrade proposal\",
+   \"summary\": \"Test upgrade proposal summary\"
+          }" > upgrade_proposal.json;
+echo ${VALPASS} | empowerd tx gov submit-proposal upgrade_proposal.json --from validator --yes --chain-id emp-devnet-1;
+sleep 5;
+PROP_ID=$(empowerd q gov proposals --reverse --limit 1 --output json | jq -r ".proposals[0].id");
+
+# Vote and wait for proposal to pass
+echo "Voting on proposal $PROP_ID";
+echo ${VALPASS} | empowerd tx gov vote ${PROP_ID} yes --from validator --yes --chain-id emp-devnet-1;
+sleep 30;
+
+# Double check that the proposal went through
+GOV_RESULT=$(empowerd q gov proposal ${PROP_ID} --output json | jq -r ".status");
+if [ "$GOV_RESULT" != "PROPOSAL_STATUS_PASSED" ]; then \
+  echo "Error: Proposal not passed: $GOV_RESULT";
+  exit 1;
+fi;
+
+# Wait for upgrade height to come around
+max_upgrade_wait_attempts=30;
+upgrade_wait_count=0;
+while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ] && [ $upgrade_wait_count -lt $max_upgrade_wait_attempts ]; do \
+    echo "Waiting for upgrade height (${UPGRADE_HEIGHT}, attempt ${upgrade_wait_count}/${max_upgrade_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    upgrade_wait_count=$((upgrade_wait_count+1));
+done;
+if [ $upgrade_wait_count -eq $max_upgrade_wait_attempts ]; then \
+    echo "Failed to wait for upgrade height ${max_upgrade_wait_attempts} attempts.";
+    cat /tmp/before_upgrade_log;
+    exit 1;
+fi;
+sleep 5; # Just for good measure
+
+# Shut down chain (it is halted at upgrade height, anyway)
+pkill empowerd;
+sleep 5; # Just for good measure
+
+# Start up the new version of empowerd
+./new_empowerd start > /tmp/after_upgrade_log 2>&1 &
+sleep 60; # Let the upgrade take
+
+after_upgrade_max_catch_up_attempts=10;
+after_upgrade_catch_up_count=0;
+while [ "`./new_empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $after_upgrade_catch_up_count -lt $after_upgrade_max_catch_up_attempts ]; do \
+    echo "Waiting to catch up after upgrade (attempt ${after_upgrade_catch_up_count}/${after_upgrade_max_catch_up_attempts}, current block: $(./new_empowerd status | jq -r ".SyncInfo.latest_block_height")";
+    sleep 5;
+    after_upgrade_catch_up_count=$((after_upgrade_catch_up_count+1));
+done;
+if [ $after_upgrade_catch_up_count -eq $after_upgrade_max_catch_up_attempts ]; then \
+    echo "Failed to catch up after the upgrade after ${after_upgrade_max_catch_up_attempts} attempts.";
+    cat /tmp/after_upgrade_log;
+    exit 1;
+fi;
+CURRENT_HEIGHT=$(./new_empowerd status | jq -r ".SyncInfo.latest_block_height");
+if [ $CURRENT_HEIGHT -le $UPGRADE_HEIGHT ]; then \
+    echo "Error: Current height ($CURRENT_HEIGHT) is less than or equal to upgrade height ($UPGRADE_HEIGHT)";
+    cat /tmp/after_upgrade_log;
+    exit 1;
+fi;
+echo "Upgrade test passed! Halted and upgraded on $UPGRADE_HEIGHT, current height: $CURRENT_HEIGHT";
\ No newline at end of file

From c0174d62806feda9cda72e360818b27db50c146e Mon Sep 17 00:00:00 2001
From: Gjermund Garaba <gjermund@garaba.net>
Date: Tue, 7 Nov 2023 14:49:37 +1100
Subject: [PATCH 8/8] Update GH workflows with workflow dispatch inputs

---
 .github/workflows/chain_genesis-export.yml | 20 ++++++++++++++++++--
 .github/workflows/chain_upgrade-test.yml   | 19 +++++++++++++++----
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/chain_genesis-export.yml b/.github/workflows/chain_genesis-export.yml
index e5118d517..f502637e3 100644
--- a/.github/workflows/chain_genesis-export.yml
+++ b/.github/workflows/chain_genesis-export.yml
@@ -2,7 +2,23 @@ name: "Export EmpowerChain Genesis"
 
 on:
   workflow_dispatch:
+    inputs:
+      chain:
+        type: choice
+        description: "Used to get a correct name for the exported genesis file"
+        options:
+          - mainnet
+          - testnet
+      chain_id:
+        description: "Chain ID"
+      state_sync_rpc:
+        description: "State sync rpc"
+      state_sync_peer:
+        description: "State sync peer"
+      current_version:
+        description: "Current version of the binary"
   # TODO: Maybe do on a cron schedule, every week or something?
+  # TODO: Potentially allow for getting some of these values from secrets, especially if we run it as a cron schedule
 
 jobs:
   export-testnet-genesis:
@@ -15,11 +31,11 @@ jobs:
         with:
           version: 0.6.30
 
-      - run: earthly --ci --no-cache --output +prep-genesis
+      - run: earthly --ci --no-cache --output +prep-genesis --CHAIN_ID=${{ github.event.inputs.chain_id }}  --STATE_SYNC_RPC=${{ github.event.inputs.state_sync_rpc }} --STATE_SYNC_PEER=${{ github.event.inputs.state_sync_peer }} --CURRENT_VERSION=${{ github.event.inputs.current_version }}
         working-directory: ./chain
 
       - name: Archive genesis file
         uses: actions/upload-artifact@v3
         with:
-          name: exported-testnet-genesis
+          name: exported-${{ github.event.inputs.chain }}-genesis
           path: chain/exported-genesis.json
\ No newline at end of file
diff --git a/.github/workflows/chain_upgrade-test.yml b/.github/workflows/chain_upgrade-test.yml
index 5b8294fa4..1c545d998 100644
--- a/.github/workflows/chain_upgrade-test.yml
+++ b/.github/workflows/chain_upgrade-test.yml
@@ -2,11 +2,22 @@ name: "Test EmpowerChain Upgrade"
 
 on:
   workflow_dispatch:
+    inputs:
+      chain:
+        type: choice
+        description: "Used to get a correct name for the exported genesis file"
+        options:
+          - mainnet
+          - testnet
+      current_version:
+        description: "Current version of the binary"
+      upgrade_name:
+        description: "Name of the upgrade"
   # TODO: Maybe do on a cron schedule, every week or something?
 
 jobs:
-  test-testnet-upgrade:
-    name: Test EmpowerChain Testnet Upgrade
+  test-upgrade:
+    name: Test EmpowerChain Upgrade
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -18,8 +29,8 @@ jobs:
       - name: Download a single artifact
         uses: actions/download-artifact@v3
         with:
-          name: exported-testnet-genesis
+          name: exported-${{ github.event.inputs.chain }}-genesis
           path: chain
 
-      - run: earthly --ci --no-cache +upgrade-test --UPGRADE_NAME=v2 # Rely on defaults for other args/env vars
+      - run: earthly --ci --no-cache +upgrade-test --CURRENT_VERSION=${{ github.event.inputs.current_version }} --UPGRADE_NAME=${{ github.event.inputs.upgrade_name }}
         working-directory: ./chain
\ No newline at end of file