From 6b98232336802bc12408431594e6df35936e343e Mon Sep 17 00:00:00 2001 From: sampocs Date: Sat, 10 Aug 2024 11:45:41 -0500 Subject: [PATCH] add relayers to integration tests framework (#1258) --- integration-tests/.env.local | 9 +++ integration-tests/Makefile | 70 ++++++++++++++----- integration-tests/docker-compose.yaml | 15 ++++ .../dockerfiles/Dockerfile.hermes | 14 ++++ .../dockerfiles/Dockerfile.relayer | 23 ++++++ integration-tests/network/configs/hermes.toml | 63 +++++++++++++++++ integration-tests/network/configs/keys.json | 25 ++++++- .../network/configs/relayer.yaml | 46 ++++++++++++ integration-tests/network/scripts/config.sh | 43 ++---------- .../network/scripts/constants.sh | 15 ++++ .../network/scripts/init-chain.sh | 69 ++++++++++++------ .../network/scripts/init-node.sh | 1 + .../network/scripts/start-hermes.sh | 49 +++++++++++++ .../network/scripts/start-relayer.sh | 59 ++++++++++++++++ integration-tests/network/scripts/utils.sh | 28 ++++++++ .../network/templates/relayer.yaml | 67 ++++++++++++++++++ .../network/templates/validator.yaml | 17 +++-- integration-tests/network/values.yaml | 15 +++- 18 files changed, 536 insertions(+), 92 deletions(-) create mode 100644 integration-tests/.env.local create mode 100644 integration-tests/docker-compose.yaml create mode 100644 integration-tests/dockerfiles/Dockerfile.hermes create mode 100644 integration-tests/dockerfiles/Dockerfile.relayer create mode 100644 integration-tests/network/configs/hermes.toml create mode 100644 integration-tests/network/configs/relayer.yaml create mode 100644 integration-tests/network/scripts/constants.sh create mode 100644 integration-tests/network/scripts/start-hermes.sh create mode 100644 integration-tests/network/scripts/start-relayer.sh create mode 100644 integration-tests/network/scripts/utils.sh create mode 100644 integration-tests/network/templates/relayer.yaml diff --git a/integration-tests/.env.local b/integration-tests/.env.local new file mode 100644 index 0000000000..ce09a61175 --- /dev/null +++ b/integration-tests/.env.local @@ -0,0 +1,9 @@ +HOSTNAME=stride-validator-0 +CHAIN_NAME=stride +CHAIN_HOME=.stride +BINARY=strided +DENOM=ustrd +DENOM_DECIMALS=6 +NUM_VALIDATORS=3 + +API_ENDPOINT=http://api:8000 \ No newline at end of file diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 202ef9c87d..e7b105cfce 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -9,38 +9,74 @@ VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin PYTHON := $(VENV_BIN)/python HELM_CHART=network +PLATFORM=linux/amd64 +GCR_REPO = gcr.io/stride-nodes/integration-tests + +# Builds and pushes a docker image +# args: [image-file-suffix] [context] [image-name] +# e.g. $(call build_and_push_docker,stride,.,chains/stride:latest +define build_and_push_docker + @echo "Building docker image: $(1)" + @$(DOCKER) buildx build --platform $(PLATFORM) --tag stride-tests:$(1) -f dockerfiles/Dockerfile.$(1) $(2) + @$(DOCKER) tag stride-tests:$(1) $(GCR_REPO)/$(3) + @echo "Pushing image to GCR: $(GCR_REPO)/$(3)" + @$(DOCKER) push $(GCR_REPO)/$(3) +endef + +check-empty-namespace: + @POD_COUNT=$$($(KUBECTL) get pods --no-headers -n $(K8S_NAMESPACE) | wc -l); \ + if [ $$POD_COUNT -eq 0 ]; then exit 0; else exit 1; fi + +wait-for-startup: + @echo "---" && sleep 1 + @printf "Waiting for network to startup..." + @elapsed=0; \ + while true; do \ + not_ready_pods=$$($(KUBECTL) get pods --no-headers -n $(K8S_NAMESPACE) | grep -v '1/1 *Running' | wc -l); \ + if [ $$not_ready_pods -eq 0 ]; then \ + printf "Ready! 🚀\n"; \ + break; \ + fi; \ + if [ $$elapsed -eq 30 ]; then \ + printf "\nThe network's taking longer than expected to startup. Please investigate\n"; \ + exit 1; \ + fi; \ + sleep 1 && printf "."; \ + elapsed=$$((elapsed + 1)); \ + done python-install: conda create --name $(VENV_NAME) python=3.11 -y $(PYTHON) -m pip install -r api/requirements.txt -start-api: - @$(PYTHON) -m uvicorn api.main:app --proxy-headers +start-api: docker-bridge + @$(DOCKER) compose up api build-api: - @echo "Building docker image: api" - @$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api - @$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest - @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest + $(call build_and_push_docker,api,api,api:latest) build-stride: - @echo "Building docker image: stride-validator" @$(DOCKER) buildx build --platform linux/amd64 --tag core:stride .. - @$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride . - @$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest - @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest + $(call build_and_push_docker,stride,.,chains/stride:latest) build-cosmos: - @echo "Building docker image" - @$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos . - @$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 - @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 + $(call build_and_push_docker,cosmos,.,chains/cosmoshub:v18.1.0) + +build-relayer: + $(call build_and_push_docker,relayer,.,relayer:v2.5.2) + +build-hermes: + $(call build_and_push_docker,hermes,.,hermes:v1.9.0) + +local-init-chain: + @$(DOCKER) compose up --abort-on-container-exit; \ + EXIT_CODE=$$?; \ + $(DOCKER) compose down; \ + exit $$EXIT_CODE start: @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE) + @$(MAKE) wait-for-startup stop: @$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE) diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml new file mode 100644 index 0000000000..d26fdb1be6 --- /dev/null +++ b/integration-tests/docker-compose.yaml @@ -0,0 +1,15 @@ +services: + api: + image: gcr.io/stride-nodes/integration-tests/api:latest + ports: + - '8000:8000' + init-chain: + image: gcr.io/stride-nodes/integration-tests/chains/stride:latest + depends_on: + - api + volumes: + - ./network/scripts:/home/validator/scripts + - ./network/configs:/home/validator/configs + env_file: + - .env.local + command: ["bash", "scripts/init-chain.sh"] diff --git a/integration-tests/dockerfiles/Dockerfile.hermes b/integration-tests/dockerfiles/Dockerfile.hermes new file mode 100644 index 0000000000..639b80ced3 --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.hermes @@ -0,0 +1,14 @@ +FROM rust:1.71-buster as builder + +COPY --from=informalsystems/hermes:v1.9.0 /usr/bin/hermes /usr/bin/hermes + +RUN apt-get update \ + && apt-get install -y iputils-ping ca-certificates libssl-dev bash vim curl jq \ + && addgroup --gid 1000 hermes \ + && adduser --system --home /home/hermes --disabled-password --disabled-login \ + --uid 1000 --ingroup hermes hermes + +USER hermes +WORKDIR /home/hermes + +CMD ["bash", "scripts/start-hermes.sh"] \ No newline at end of file diff --git a/integration-tests/dockerfiles/Dockerfile.relayer b/integration-tests/dockerfiles/Dockerfile.relayer new file mode 100644 index 0000000000..ef2875c1ce --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.relayer @@ -0,0 +1,23 @@ +FROM golang:1.21-alpine3.17 AS builder + +WORKDIR /src/ + +ENV COMMIT_HASH=v2.5.2 + +RUN apk add --update git make gcc linux-headers libc-dev eudev-dev +RUN git clone https://github.com/cosmos/relayer.git \ + && cd relayer \ + && git checkout ${COMMIT_HASH} \ + && CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install + +FROM alpine:3.17 + +COPY --from=builder /go/bin/rly /usr/local/bin/ +RUN apk add --no-cache --update bash vim curl iputils jq yq \ + && addgroup -g 1000 relayer \ + && adduser -S -h /home/relayer -D relayer -u 1000 -G relayer + +USER relayer +WORKDIR /home/relayer + +CMD ["bash", "scripts/start-relayer.sh"] \ No newline at end of file diff --git a/integration-tests/network/configs/hermes.toml b/integration-tests/network/configs/hermes.toml new file mode 100644 index 0000000000..273cd1d328 --- /dev/null +++ b/integration-tests/network/configs/hermes.toml @@ -0,0 +1,63 @@ +[global] +log_level = 'debug' + +[mode] +[mode.clients] +enabled = true +refresh = true +misbehaviour = true + +[mode.connections] +enabled = true + +[mode.channels] +enabled = true + +[mode.packets] +enabled = true +clear_interval = 100 +clear_on_start = true + +[rest] +enabled = false +host = '0.0.0.0' +port = 3000 + +[telemetry] +enabled = false +host = '127.0.0.1' +port = 3001 + +[[chains]] +id = 'stride-test-1' +rpc_addr = 'http://stride-validator.integration.svc:26657' +grpc_addr = 'http://stride-validator.integration.svc:9090' +event_source = { mode = 'push', url = 'ws://stride-validator.integration.svc:26657/websocket', batch_delay = '500ms' } +rpc_timeout = '10s' +account_prefix = 'stride' +key_name = 'stride' +store_prefix = 'ibc' +# TODO: investigate why this has to be set so high +gas_price = { price = 1, denom = 'ustrd' } +gas_multiplier = 1.3 +clock_drift = '10s' +max_block_time = '10s' +address_type = { derivation = 'cosmos' } +memo_prefix = 'stride-tests' + +[[chains]] +id = 'cosmoshub-test-1' +rpc_addr = 'http://cosmoshub-validator.integration.svc:26657' +grpc_addr = 'http://cosmoshub-validator.integration.svc:9090' +event_source = { mode = 'push', url = 'ws://cosmoshub-validator.integration.svc:26657/websocket', batch_delay = '500ms' } +rpc_timeout = '10s' +account_prefix = 'cosmos' +key_name = 'cosmoshub' +store_prefix = 'ibc' +# TODO: investigate why this has to be set so high +gas_price = { price = 1, denom = 'uatom' } +gas_multiplier = 1.3 +clock_drift = '10s' +max_block_time = '10s' +address_type = { derivation = 'cosmos' } +memo_prefix = 'stride-tests' diff --git a/integration-tests/network/configs/keys.json b/integration-tests/network/configs/keys.json index 33aace07f4..31c0ace230 100644 --- a/integration-tests/network/configs/keys.json +++ b/integration-tests/network/configs/keys.json @@ -1,4 +1,13 @@ { + + "admin": { + "name": "admin", + "mnemonic": "tone cause tribe this switch near host damage idle fragile antique tail soda alien depth write wool they rapid unfold body scan pledge soft" + }, + "faucet": { + "name": "faucet", + "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film" + }, "validators": [ { "name": "val1", @@ -21,10 +30,20 @@ "mnemonic": "crime lumber parrot enforce chimney turtle wing iron scissors jealous indicate peace empty game host protect juice submit motor cause second picture nuclear area" } ], - "faucet": [ + "relayers": [ { - "name": "faucet", - "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film" + "name": "rly1", + "mnemonic": "fringe secret hair noise royal inform remove release mother fish swamp swim piece later course glimpse sing very clutch velvet prefer hover common mass" + }, + { + "name": "rly2", + "mnemonic": "floor chicken advance degree paper arch sugar kit torch auction flash fish attitude clay museum treat favorite clinic verify dose safe bright vessel east" + } + ], + "users": [ + { + "name": "user1", + "mnemonic": "brief play describe burden half aim soccer carbon hope wait output play vacuum joke energy crucial output mimic cruise brother document rail anger leaf" } ] } diff --git a/integration-tests/network/configs/relayer.yaml b/integration-tests/network/configs/relayer.yaml new file mode 100644 index 0000000000..352d1ac6ca --- /dev/null +++ b/integration-tests/network/configs/relayer.yaml @@ -0,0 +1,46 @@ +global: + api-listen-addr: :5183 + timeout: 10s + memo: stride-tests + light-cache-size: 20 +chains: + stride: + type: cosmos + value: + key: stride + chain-id: stride-test-1 + rpc-addr: http://stride-validator.integration.svc:26657 + account-prefix: stride + keyring-backend: test + gas-adjustment: 1.3 + # TODO: investigate why this has to be set so high + gas-prices: 1ustrd + coin-type: 118 + debug: false + timeout: 20s + output-format: json + sign-mode: direct + min-loop-duration: 1s + cosmoshub: + type: cosmos + value: + key: cosmoshub + chain-id: cosmoshub-test-1 + rpc-addr: http://cosmoshub-validator.integration.svc:26657 + account-prefix: cosmos + keyring-backend: test + gas-adjustment: 1.3 + # TODO: investigate why this has to be set so high + gas-prices: 1uatom + coin-type: 118 + min-loop-duration: 1s + debug: false + timeout: 20s + output-format: json + sign-mode: direct +paths: + stride-cosmoshub: + src: + chain-id: stride-test-1 + dst: + chain-id: cosmoshub-test-1 \ No newline at end of file diff --git a/integration-tests/network/scripts/config.sh b/integration-tests/network/scripts/config.sh index a3c8d2da8a..ff96626524 100644 --- a/integration-tests/network/scripts/config.sh +++ b/integration-tests/network/scripts/config.sh @@ -1,28 +1,12 @@ #!/bin/bash -# Override the hostname only when in local mode -if [[ "$HOSTNAME" != *"validator"* ]]; then - HOSTNAME=validator-0 -fi - -SCRIPTS_DIR=scripts -CONFIG_DIR=configs - -VALIDATOR_KEYS_DIR=validator-keys -NODE_KEYS_DIR=node-keys -NODE_IDS_DIR=node-ids -GENESIS_DIR=genesis -KEYS_FILE=${CONFIG_DIR}/keys.json +set -eu +source scripts/constants.sh POD_INDEX=${HOSTNAME##*-} VALIDATOR_INDEX=$((POD_INDEX+1)) VALIDATOR_NAME=val${VALIDATOR_INDEX} -API_ENDPOINT=http://api.integration.svc:8000 - -PEER_PORT=26656 -RPC_PORT=26657 - # Redefined to confirm they're set CHAIN_NAME=${CHAIN_NAME} CHAIN_HOME=${CHAIN_HOME} @@ -36,6 +20,8 @@ CHAIN_ID=${CHAIN_NAME}-test-1 BLOCK_TIME=1s VALIDATOR_BALANCE=10000000${MICRO_DENOM_ZERO_PAD} VALIDATOR_STAKE=1000000${MICRO_DENOM_ZERO_PAD} +RELAYER_BALANCE=10000000${MICRO_DENOM_ZERO_PAD} +USER_BALANCE=10000000${MICRO_DENOM_ZERO_PAD} DEPOSIT_PERIOD="30s" VOTING_PERIOD="30s" @@ -43,23 +29,4 @@ EXPEDITED_VOTING_PERIOD="29s" UNBONDING_TIME="240s" STRIDE_DAY_EPOCH_DURATION="140s" -STRIDE_EPOCH_EPOCH_DURATION="35s" - -# Wait for API server to start -wait_for_api() { - api_endpoint="$1" - until [[ $(curl -o /dev/null -s -w "%{http_code}\n" "${api_endpoint}/status") -eq 200 ]]; do - echo "Waiting for API to start..." - sleep 2 - done -} - -# Wait for node to start -wait_for_node() { - chain_name="$1" - rpc_endpoint="http://${chain_name}-validator.integration.svc:26657/status" - until [[ $(curl -o /dev/null -s $rpc_endpoint | jq '.result.sync_info.catching_up') == "false" ]]; do - echo "Waiting for $chain_name to start..." - sleep 2 - done -} \ No newline at end of file +STRIDE_EPOCH_EPOCH_DURATION="35s" \ No newline at end of file diff --git a/integration-tests/network/scripts/constants.sh b/integration-tests/network/scripts/constants.sh new file mode 100644 index 0000000000..682fb9b329 --- /dev/null +++ b/integration-tests/network/scripts/constants.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +SCRIPTS_DIR=scripts +CONFIG_DIR=configs + +VALIDATOR_KEYS_DIR=validator-keys +NODE_KEYS_DIR=node-keys +NODE_IDS_DIR=node-ids +GENESIS_DIR=genesis +KEYS_FILE=${CONFIG_DIR}/keys.json + +PEER_PORT=26656 +RPC_PORT=26657 + +API_ENDPOINT=${API_ENDPOINT:-'http://api.integration.svc:8000'} diff --git a/integration-tests/network/scripts/init-chain.sh b/integration-tests/network/scripts/init-chain.sh index ccfc6a24e3..3169c0421c 100644 --- a/integration-tests/network/scripts/init-chain.sh +++ b/integration-tests/network/scripts/init-chain.sh @@ -2,16 +2,7 @@ set -eu source scripts/config.sh - -LOCAL_MODE=${1:-false} - -# If this is being run locally, don't overwrite the main chain folder -if [[ "$LOCAL_MODE" == "true" ]]; then - CHAIN_HOME=state - rm -rf state - BINARY="$BINARY --home $CHAIN_HOME" - API_ENDPOINT="http://localhost:8000" -fi +source scripts/utils.sh # Wait for API server to start wait_for_api $API_ENDPOINT @@ -47,6 +38,22 @@ upload_shared_file() { -F "file=@$file_path" && echo } +# Helper function to add an account to the keyring and genesis +# The key arg is a JSON string with "name" and "mnemonic" fields +add_genesis_account() { + key="$1" + balance="$2" + + echo $key + name=$(echo $key | jq -r '.name') + mnemonic=$(echo $key | jq -r '.mnemonic') + + echo "$mnemonic" | $BINARY keys add $name --recover + address=$($BINARY keys show $name -a) + + $BINARY $chain_genesis_command add-genesis-account $address $balance +} + # Adds each validator to the genesis file, and also saves down the public keys # which are needed for ICS # Each validators public private key and node ID are saved in the API @@ -55,14 +62,11 @@ add_validators() { validator_public_keys="" for (( i=1; i <= $NUM_VALIDATORS; i++ )); do - # Extract the validator name and mnemonic from keys.json - validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" ${KEYS_FILE}) - name=$(echo $validator_config | jq -r '.name') - mnemonic=$(echo $validator_config | jq -r '.mnemonic') + # Add the validator account to the keyring and genesis + validator_key=$(jq -r '.validators[$index]' --argjson index "$((i-1))" ${KEYS_FILE}) + name=$(echo $validator_key | jq -r '.name') - # Add the key to the main keyring the the validator's sub-keyring - echo "$mnemonic" | $BINARY keys add $name --recover - address=$($BINARY keys show $name -a) + add_genesis_account "$validator_key" "${VALIDATOR_BALANCE}${DENOM}" # Use a separate directory for the non-main nodes so we can generate unique validator keys if [[ "$i" == "1" ]]; then @@ -72,10 +76,6 @@ add_validators() { $BINARY init $name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null fi - # Add the genesis account - genesis_balance=${VALIDATOR_BALANCE}${DENOM} - $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance - # Save the node IDs and keys to the API $BINARY tendermint show-node-id --home ${validator_home} > node_id.txt upload_shared_file node_id.txt ${NODE_IDS_DIR}/${CHAIN_NAME}/${name}.txt @@ -86,14 +86,34 @@ add_validators() { validator_public_keys+="$(jq -r '.pub_key.value' ${validator_home}/config/priv_validator_key.json)," done - # For non-stride nodes, generate and collect the validator gentx (for the main node only) + # For non-stride nodes, generate the validator gentx (for the main node only) # The other validators will be created after startup if [[ "$CHAIN_NAME" != "stride" ]]; then $BINARY $chain_genesis_command gentx val1 ${VALIDATOR_STAKE}${DENOM} --chain-id $CHAIN_ID - $BINARY $chain_genesis_command collect-gentxs fi } +# Adds all the non-validator accounts +add_accounts() { + echo "Adding admin account..." + admin_key=$(cat $KEYS_FILE | jq -c '.admin') + add_genesis_account "$admin_key" ${USER_BALANCE}${DENOM} + + echo "Adding faucet account..." + faucet_key=$(cat $KEYS_FILE | jq -c '.faucet') + add_genesis_account "$faucet_key" ${USER_BALANCE}${DENOM} + + echo "Adding relayer accounts..." + jq -c '.relayers[]' $KEYS_FILE | while IFS= read -r relayer_key; do + add_genesis_account "$relayer_key" ${RELAYER_BALANCE}${DENOM} + done + + echo "Adding user accounts..." + jq -c '.users[]' $KEYS_FILE | while IFS= read -r user_keys; do + add_genesis_account "$user_keys" ${RELAYER_BALANCE}${DENOM} + done +} + # Updates the genesis config with defaults update_default_genesis() { echo "Updating genesis.json with defaults..." @@ -118,6 +138,7 @@ update_stride_genesis() { jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json $BINARY add-consumer-section --validator-public-keys $validator_public_keys + jq_inplace '.app_state.ccvconsumer.params.unbonding_period |= "'$UNBONDING_TIME'"' $genesis_json } # Genesis updates specific to non-stride chains @@ -136,11 +157,13 @@ main() { echo "Initializing chain..." init_config add_validators + add_accounts update_default_genesis if [[ "$CHAIN_NAME" == "stride" ]]; then update_stride_genesis else update_host_genesis + $BINARY $chain_genesis_command collect-gentxs fi save_genesis echo "Done" diff --git a/integration-tests/network/scripts/init-node.sh b/integration-tests/network/scripts/init-node.sh index ba0dd1a9f5..fc06a747aa 100644 --- a/integration-tests/network/scripts/init-node.sh +++ b/integration-tests/network/scripts/init-node.sh @@ -2,6 +2,7 @@ set -eu source scripts/config.sh +source scripts/utils.sh # Wait for API server to start wait_for_api $API_ENDPOINT diff --git a/integration-tests/network/scripts/start-hermes.sh b/integration-tests/network/scripts/start-hermes.sh new file mode 100644 index 0000000000..938d2b4766 --- /dev/null +++ b/integration-tests/network/scripts/start-hermes.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eu +source scripts/utils.sh +source scripts/constants.sh + +CHAIN_ID_A=${CHAIN_NAME_A}-test-1 +CHAIN_ID_B=${CHAIN_NAME_B}-test-1 + +wait_for_node $CHAIN_NAME_A +wait_for_node $CHAIN_NAME_B + +restore_keys() { + mnemonic_a=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_A_MNEMONIC_INDEX" ${KEYS_FILE}) + mnemonic_b=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_B_MNEMONIC_INDEX" ${KEYS_FILE}) + + echo "$mnemonic_a" > /tmp/mnemonics_a.txt + echo "$mnemonic_b" > /tmp/mnemonics_b.txt + + hermes keys add --chain $CHAIN_ID_A --mnemonic-file /tmp/mnemonics_a.txt + hermes keys add --chain $CHAIN_ID_B --mnemonic-file /tmp/mnemonics_b.txt + + rm -f /tmp/mnemonics_a.txt /tmp/mnemonics_b.txt +} + +create_path() { + # Only create a path if one does not already exist + if ! hermes query channels --chain $CHAIN_ID_A --show-counterparty | grep -q $CHAIN_ID_A; then + echo "Creating path..." + hermes create channel --a-chain $CHAIN_ID_A --b-chain $CHAIN_ID_B \ + --a-port transfer --b-port transfer --new-client-connection --yes + fi +} + +main() { + # The config is mounted from a configmap which is read-only by default + # In order to make it writeable, we need to copy it to a new location + hermes_config_file=${HOME}/.hermes/config.toml + mkdir -p $(dirname $hermes_config_file) + cp configs/hermes.toml ${hermes_config_file} + + restore_keys + create_path + + echo "Starting hermes..." + hermes start +} + +main \ No newline at end of file diff --git a/integration-tests/network/scripts/start-relayer.sh b/integration-tests/network/scripts/start-relayer.sh new file mode 100644 index 0000000000..ccfed98d6b --- /dev/null +++ b/integration-tests/network/scripts/start-relayer.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -eu +source scripts/utils.sh +source scripts/constants.sh + +CHAIN_ID_A=${CHAIN_NAME_A}-test-1 +CHAIN_ID_B=${CHAIN_NAME_B}-test-1 + +wait_for_node $CHAIN_NAME_A +wait_for_node $CHAIN_NAME_B + +relayer_config_file=${HOME}/.relayer/config/config.yaml + +restore_keys() { + mnemonic_a=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_A_MNEMONIC_INDEX" ${KEYS_FILE}) + mnemonic_b=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_B_MNEMONIC_INDEX" ${KEYS_FILE}) + + rly keys restore $CHAIN_NAME_A $CHAIN_NAME_A "$mnemonic_a" + rly keys restore $CHAIN_NAME_B $CHAIN_NAME_B "$mnemonic_b" +} + +create_path() { + # If there aren't any channels yet, create a new path + if ! rly q channels $CHAIN_NAME_A 2>/dev/null | grep -q $CHAIN_ID_A; then + echo "Creating path..." + rly tx link $PATH_NAME + else + # Otherwise, add the exising connection to the config + client_id_a=$(rly q clients $CHAIN_NAME_A | grep $CHAIN_ID_B | jq -r '.client_id') + client_id_b=$(rly q clients $CHAIN_NAME_B | grep $CHAIN_ID_A | jq -r '.client_id') + + connection_id_a=$(rly q client-connections $CHAIN_NAME_A $client_id_a | jq -r '.connections[0].id') + connection_id_b=$(rly q client-connections $CHAIN_NAME_B $client_id_b | jq -r '.connections[0].id') + + echo "Path already found, updating config..." + yq eval -i " + .paths.\"${PATH_NAME}\".src.client-id = \"$client_id_a\" | + .paths.\"${PATH_NAME}\".src.connection = \"$connection_id_a\" | + .paths.\"${PATH_NAME}\".dst.client-id = \"$client_id_b\" | + .paths.\"${PATH_NAME}\".dst.connection = \"$connection_id_b\" + " "$relayer_config_file" + fi +} + +main() { + # The config is mounted from a configmap which is read-only by default + # In order to make it writeable, we need to copy it to a new location + mkdir -p $(dirname $relayer_config_file) + cp configs/relayer.yaml ${relayer_config_file} + + restore_keys + create_path + + echo "Starting relayer..." + rly start $PATH_NAME +} + +main \ No newline at end of file diff --git a/integration-tests/network/scripts/utils.sh b/integration-tests/network/scripts/utils.sh new file mode 100644 index 0000000000..3d397b02a5 --- /dev/null +++ b/integration-tests/network/scripts/utils.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -eu + +# Wait for API server to start +wait_for_api() { + api_endpoint="$1" + until [[ $(curl -o /dev/null -s -w "%{http_code}\n" "${api_endpoint}/status") -eq 200 ]]; do + echo "Waiting for API to start..." + sleep 2 + done +} + +# Wait for node to start +wait_for_node() { + chain_name="$1" + rpc_endpoint="http://${chain_name}-validator.integration.svc:26657/status" + + # Wait for the node to be caught up and confirm it's at least on the 2nd block + until + response=$(curl -s "$rpc_endpoint") + catching_up=$(echo "$response" | jq -r '.result.sync_info.catching_up') + latest_block=$(echo "$response" | jq -r '.result.sync_info.latest_block_height') + [[ $catching_up == "false" && $latest_block -gt 2 ]] + do + echo "Waiting for $chain_name to start..." + sleep 2 + done +} \ No newline at end of file diff --git a/integration-tests/network/templates/relayer.yaml b/integration-tests/network/templates/relayer.yaml new file mode 100644 index 0000000000..4b3785c4ef --- /dev/null +++ b/integration-tests/network/templates/relayer.yaml @@ -0,0 +1,67 @@ +{{- $root := .Values -}} +{{- range $index, $relayer := .Values.relayers }} + {{- $appName := printf "%s-%s" $relayer.type $relayer.name -}} + + {{- $image := "" -}} + {{- if eq $relayer.type "relayer" -}} + {{- $image = $root.images.relayer -}} + {{- else if eq $relayer.type "hermes" -}} + {{- $image = $root.images.hermes -}} + {{- else -}} + {{- fail (printf "Unknown relayer type: %s" $relayer.type) -}} + {{- end -}} + + {{- $chainMnemonicIndexA := mul $index 2 }} + {{- $chainMnemonicIndexB := add $chainMnemonicIndexA 1 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $appName }} + namespace: {{ $root.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ $appName }} + template: + metadata: + labels: + app: {{ $appName }} + spec: + containers: + - name: {{ $appName }} + image: {{ $image }} + imagePullPolicy: Always + env: + - name: PATH_NAME + value: {{ $relayer.name }} + - name: CHAIN_NAME_A + value: {{ $relayer.chainA }} + - name: CHAIN_NAME_B + value: {{ $relayer.chainB }} + - name: CHAIN_A_MNEMONIC_INDEX + value: "{{ $chainMnemonicIndexA }}" + - name: CHAIN_B_MNEMONIC_INDEX + value: "{{ $chainMnemonicIndexB }}" + resources: + limits: + cpu: "200m" + memory: "1000M" + requests: + cpu: "200m" + memory: "1000M" + volumeMounts: + - name: configs + mountPath: "/home/{{ $relayer.type }}/configs" + - name: scripts + mountPath: "/home/{{ $relayer.type }}/scripts" + volumes: + - name: configs + configMap: + name: configs + - name: scripts + configMap: + name: scripts +--- +{{- end }} \ No newline at end of file diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index a278208ed6..fdf269fc14 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -1,3 +1,5 @@ +{{- $root := .Values -}} + {{- define "chain.env" -}} - name: CHAIN_NAME value: {{ .name }} @@ -17,19 +19,16 @@ fieldPath: metadata.namespace {{- end -}} -{{- $namespace := .Values.namespace -}} -{{- $imageRepo := .Values.chainImagesRepo -}} - {{- range $chain := .Values.chains }} -{{- $appName := printf "%s-validator" $chain.name -}} -{{- $chainHomeDirectory := printf "/home/validator/%s" $chain.home -}} -{{- $image := printf "%s/%s:%s" $imageRepo $chain.name $chain.version -}} + {{- $appName := printf "%s-validator" $chain.name -}} + {{- $chainHomeDirectory := printf "/home/validator/%s" $chain.home -}} + {{- $image := printf "%s/%s:%s" $root.images.chains $chain.name $chain.version -}} --- apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ $appName }} - namespace: {{ $namespace }} + namespace: {{ $root.namespace }} spec: replicas: {{ $chain.numValidators }} selector: @@ -121,7 +120,7 @@ apiVersion: v1 kind: Service metadata: name: {{ $appName }} - namespace: {{ $namespace }} + namespace: {{ $root.namespace }} spec: clusterIP: None selector: @@ -144,7 +143,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ $appName }} - namespace: {{ $namespace }} + namespace: {{ $root.namespace }} annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" diff --git a/integration-tests/network/values.yaml b/integration-tests/network/values.yaml index 8bf230e8af..45fff8a88b 100644 --- a/integration-tests/network/values.yaml +++ b/integration-tests/network/values.yaml @@ -1,5 +1,9 @@ namespace: integration -chainImagesRepo: gcr.io/stride-nodes/integration-tests/chains + +images: + chains: gcr.io/stride-nodes/integration-tests/chains + relayer: gcr.io/stride-nodes/integration-tests/relayer:v2.5.2 + hermes: gcr.io/stride-nodes/integration-tests/hermes:v1.9.0 chains: - name: stride @@ -18,4 +22,11 @@ chains: home: .gaia denom: uatom decimals: 6 - command: ["gaiad", "start"] \ No newline at end of file + command: ["gaiad", "start"] + +# type can be either "relayer" or "hermes" +relayers: + - name: stride-cosmoshub + type: relayer + chainA: stride + chainB: cosmoshub \ No newline at end of file