add relayers to integration tests framework (#1258)

integration-tests/.env.local

@@ -0,0 +1,9 @@
+HOSTNAME=stride-validator-0
+CHAIN_NAME=stride
+CHAIN_HOME=.stride
+BINARY=strided
+DENOM=ustrd
+DENOM_DECIMALS=6
+NUM_VALIDATORS=3
+
+API_ENDPOINT=http://api:8000

integration-tests/Makefile

@@ -9,38 +9,74 @@ VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin
 PYTHON := $(VENV_BIN)/python
 
 HELM_CHART=network
+PLATFORM=linux/amd64
+GCR_REPO = gcr.io/stride-nodes/integration-tests
+
+# Builds and pushes a docker image
+# args: [image-file-suffix] [context] [image-name] 
+# e.g. $(call build_and_push_docker,stride,.,chains/stride:latest
+define build_and_push_docker
+	@echo "Building docker image: $(1)"
+	@$(DOCKER) buildx build --platform $(PLATFORM) --tag stride-tests:$(1) -f dockerfiles/Dockerfile.$(1) $(2)
+	@$(DOCKER) tag stride-tests:$(1) $(GCR_REPO)/$(3)
+	@echo "Pushing image to GCR: $(GCR_REPO)/$(3)"
+	@$(DOCKER) push $(GCR_REPO)/$(3)
+endef
+
+check-empty-namespace:
+	@POD_COUNT=$$($(KUBECTL) get pods --no-headers -n $(K8S_NAMESPACE) | wc -l); \
+	if [ $$POD_COUNT -eq 0 ]; then exit 0; else exit 1; fi
+
+wait-for-startup:
+	@echo "---" && sleep 1
+	@printf "Waiting for network to startup..."
+	@elapsed=0; \
+	while true; do \
+		not_ready_pods=$$($(KUBECTL) get pods --no-headers -n $(K8S_NAMESPACE) | grep -v '1/1 *Running' | wc -l); \
+		if [ $$not_ready_pods -eq 0 ]; then \
+			printf "Ready! 🚀\n"; \
+			break; \
+		fi; \
+		if [ $$elapsed -eq 30 ]; then \
+			printf "\nThe network's taking longer than expected to startup. Please investigate\n"; \
+			exit 1; \
+		fi; \
+		sleep 1 && printf "."; \
+		elapsed=$$((elapsed + 1)); \
+	done
 
 python-install:
 	conda create --name $(VENV_NAME) python=3.11 -y
 	$(PYTHON) -m pip install -r api/requirements.txt
 
-start-api:
-	@$(PYTHON) -m uvicorn api.main:app --proxy-headers 
+start-api: docker-bridge
+	@$(DOCKER) compose up api
 
 build-api:
-	@echo "Building docker image: api"
-	@$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api
-	@$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest
-	@echo "Pushing image to GCR"
-	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest
+	$(call build_and_push_docker,api,api,api:latest)
 
 build-stride:
-	@echo "Building docker image: stride-validator"
 	@$(DOCKER) buildx build --platform linux/amd64 --tag core:stride ..
-	@$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride .
-	@$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest
-	@echo "Pushing image to GCR"
-	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest
+	$(call build_and_push_docker,stride,.,chains/stride:latest)
 
 build-cosmos:
-	@echo "Building docker image"
-	@$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos .
-	@$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
-	@echo "Pushing image to GCR"
-	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
+	$(call build_and_push_docker,cosmos,.,chains/cosmoshub:v18.1.0)
+
+build-relayer:
+	$(call build_and_push_docker,relayer,.,relayer:v2.5.2)
+
+build-hermes:
+	$(call build_and_push_docker,hermes,.,hermes:v1.9.0)
+
+local-init-chain: 
+	@$(DOCKER) compose up --abort-on-container-exit; \
+	EXIT_CODE=$$?; \
+	$(DOCKER) compose down; \
+	exit $$EXIT_CODE
 
 start:
 	@$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE)
+	@$(MAKE) wait-for-startup
 
 stop:
 	@$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE)

integration-tests/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  api:
+    image: gcr.io/stride-nodes/integration-tests/api:latest 
+    ports:
+      - '8000:8000'
+  init-chain:
+    image: gcr.io/stride-nodes/integration-tests/chains/stride:latest
+    depends_on: 
+      - api
+    volumes:
+      - ./network/scripts:/home/validator/scripts
+      - ./network/configs:/home/validator/configs
+    env_file:
+      - .env.local
+    command: ["bash", "scripts/init-chain.sh"]

integration-tests/dockerfiles/Dockerfile.hermes

@@ -0,0 +1,14 @@
+FROM rust:1.71-buster as builder
+
+COPY --from=informalsystems/hermes:v1.9.0 /usr/bin/hermes /usr/bin/hermes
+
+RUN apt-get update \
+    && apt-get install -y iputils-ping ca-certificates libssl-dev bash vim curl jq \
+    && addgroup --gid 1000 hermes \
+    && adduser --system --home /home/hermes --disabled-password --disabled-login \
+    --uid 1000 --ingroup hermes hermes
+
+USER hermes
+WORKDIR /home/hermes
+
+CMD ["bash", "scripts/start-hermes.sh"]

integration-tests/dockerfiles/Dockerfile.relayer

@@ -0,0 +1,23 @@
+FROM golang:1.21-alpine3.17 AS builder
+
+WORKDIR /src/
+
+ENV COMMIT_HASH=v2.5.2
+
+RUN apk add --update git make gcc linux-headers libc-dev eudev-dev 
+RUN git clone https://github.com/cosmos/relayer.git \
+    && cd relayer \
+    && git checkout ${COMMIT_HASH} \
+    && CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install
+
+FROM alpine:3.17
+
+COPY --from=builder /go/bin/rly /usr/local/bin/
+RUN apk add --no-cache --update bash vim curl iputils jq yq \
+    && addgroup -g 1000 relayer \
+    && adduser -S -h /home/relayer -D relayer -u 1000 -G relayer 
+
+USER relayer
+WORKDIR /home/relayer
+
+CMD ["bash", "scripts/start-relayer.sh"]

integration-tests/network/configs/hermes.toml

@@ -0,0 +1,63 @@
+[global]
+log_level = 'debug'
+
+[mode]
+[mode.clients]
+enabled = true
+refresh = true
+misbehaviour = true
+
+[mode.connections]
+enabled = true
+
+[mode.channels]
+enabled = true
+
+[mode.packets]
+enabled = true
+clear_interval = 100
+clear_on_start = true
+
+[rest]
+enabled = false
+host = '0.0.0.0'
+port = 3000
+
+[telemetry]
+enabled = false
+host = '127.0.0.1'
+port = 3001
+
+[[chains]]
+id = 'stride-test-1'
+rpc_addr = 'http://stride-validator.integration.svc:26657'
+grpc_addr = 'http://stride-validator.integration.svc:9090'
+event_source = { mode = 'push', url = 'ws://stride-validator.integration.svc:26657/websocket', batch_delay = '500ms' }
+rpc_timeout = '10s'
+account_prefix = 'stride'
+key_name = 'stride'
+store_prefix = 'ibc'
+# TODO: investigate why this has to be set so high
+gas_price = { price = 1, denom = 'ustrd' }
+gas_multiplier = 1.3
+clock_drift = '10s'
+max_block_time = '10s'
+address_type = { derivation = 'cosmos' }
+memo_prefix = 'stride-tests'
+
+[[chains]]
+id = 'cosmoshub-test-1'
+rpc_addr = 'http://cosmoshub-validator.integration.svc:26657'
+grpc_addr = 'http://cosmoshub-validator.integration.svc:9090'
+event_source = { mode = 'push', url = 'ws://cosmoshub-validator.integration.svc:26657/websocket', batch_delay = '500ms' }
+rpc_timeout = '10s'
+account_prefix = 'cosmos'
+key_name = 'cosmoshub'
+store_prefix = 'ibc'
+# TODO: investigate why this has to be set so high
+gas_price = { price = 1, denom = 'uatom' }
+gas_multiplier = 1.3
+clock_drift = '10s'
+max_block_time = '10s'
+address_type = { derivation = 'cosmos' }
+memo_prefix = 'stride-tests'

integration-tests/network/configs/keys.json

@@ -1,4 +1,13 @@
 {
+
+    "admin": {
+      "name": "admin",
+      "mnemonic": "tone cause tribe this switch near host damage idle fragile antique tail soda alien depth write wool they rapid unfold body scan pledge soft"
+    },
+    "faucet": {
+      "name": "faucet",
+      "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film"
+    },
     "validators": [
       {
         "name": "val1",
@@ -21,10 +30,20 @@
         "mnemonic": "crime lumber parrot enforce chimney turtle wing iron scissors jealous indicate peace empty game host protect juice submit motor cause second picture nuclear area"
       }
     ],
-    "faucet": [
+    "relayers": [
       {
-        "name": "faucet",
-        "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film"
+        "name": "rly1",
+        "mnemonic": "fringe secret hair noise royal inform remove release mother fish swamp swim piece later course glimpse sing very clutch velvet prefer hover common mass"
+      },
+      {
+        "name": "rly2",
+        "mnemonic": "floor chicken advance degree paper arch sugar kit torch auction flash fish attitude clay museum treat favorite clinic verify dose safe bright vessel east"
+      }
+    ],
+    "users": [
+      {
+        "name": "user1",
+        "mnemonic": "brief play describe burden half aim soccer carbon hope wait output play vacuum joke energy crucial output mimic cruise brother document rail anger leaf"
       }
     ]
   }

integration-tests/network/configs/relayer.yaml

@@ -0,0 +1,46 @@
+global:
+  api-listen-addr: :5183
+  timeout: 10s
+  memo: stride-tests
+  light-cache-size: 20
+chains:
+  stride:
+    type: cosmos
+    value:
+      key: stride
+      chain-id: stride-test-1
+      rpc-addr: http://stride-validator.integration.svc:26657
+      account-prefix: stride
+      keyring-backend: test
+      gas-adjustment: 1.3
+      # TODO: investigate why this has to be set so high
+      gas-prices: 1ustrd
+      coin-type: 118
+      debug: false
+      timeout: 20s
+      output-format: json
+      sign-mode: direct
+      min-loop-duration: 1s
+  cosmoshub:
+    type: cosmos
+    value:
+      key: cosmoshub
+      chain-id: cosmoshub-test-1
+      rpc-addr: http://cosmoshub-validator.integration.svc:26657
+      account-prefix: cosmos
+      keyring-backend: test
+      gas-adjustment: 1.3
+      # TODO: investigate why this has to be set so high
+      gas-prices: 1uatom
+      coin-type: 118
+      min-loop-duration: 1s
+      debug: false
+      timeout: 20s
+      output-format: json
+      sign-mode: direct
+paths:
+  stride-cosmoshub:
+    src:
+      chain-id: stride-test-1
+    dst:
+      chain-id: cosmoshub-test-1

integration-tests/network/scripts/config.sh

@@ -1,28 +1,12 @@
 #!/bin/bash
 
-# Override the hostname only when in local mode
-if [[ "$HOSTNAME" != *"validator"* ]]; then
-    HOSTNAME=validator-0
-fi
-
-SCRIPTS_DIR=scripts
-CONFIG_DIR=configs
-
-VALIDATOR_KEYS_DIR=validator-keys
-NODE_KEYS_DIR=node-keys
-NODE_IDS_DIR=node-ids
-GENESIS_DIR=genesis
-KEYS_FILE=${CONFIG_DIR}/keys.json
+set -eu
+source scripts/constants.sh
 
 POD_INDEX=${HOSTNAME##*-}
 VALIDATOR_INDEX=$((POD_INDEX+1))
 VALIDATOR_NAME=val${VALIDATOR_INDEX}
 
-API_ENDPOINT=http://api.integration.svc:8000
-
-PEER_PORT=26656
-RPC_PORT=26657
-
 # Redefined to confirm they're set
 CHAIN_NAME=${CHAIN_NAME}
 CHAIN_HOME=${CHAIN_HOME}
@@ -36,30 +20,13 @@ CHAIN_ID=${CHAIN_NAME}-test-1
 BLOCK_TIME=1s
 VALIDATOR_BALANCE=10000000${MICRO_DENOM_ZERO_PAD}
 VALIDATOR_STAKE=1000000${MICRO_DENOM_ZERO_PAD}
+RELAYER_BALANCE=10000000${MICRO_DENOM_ZERO_PAD}
+USER_BALANCE=10000000${MICRO_DENOM_ZERO_PAD}
 
 DEPOSIT_PERIOD="30s"
 VOTING_PERIOD="30s"
 EXPEDITED_VOTING_PERIOD="29s"
 UNBONDING_TIME="240s"
 
 STRIDE_DAY_EPOCH_DURATION="140s"
-STRIDE_EPOCH_EPOCH_DURATION="35s"
-
-# Wait for API server to start
-wait_for_api() {
-    api_endpoint="$1"
-    until [[ $(curl -o /dev/null -s -w "%{http_code}\n" "${api_endpoint}/status") -eq 200 ]]; do
-        echo "Waiting for API to start..."
-        sleep 2
-    done
-}
-
-# Wait for node to start
-wait_for_node() {
-    chain_name="$1"
-    rpc_endpoint="http://${chain_name}-validator.integration.svc:26657/status"
-    until [[ $(curl -o /dev/null -s $rpc_endpoint | jq '.result.sync_info.catching_up') == "false" ]]; do
-        echo "Waiting for $chain_name to start..."
-        sleep 2
-    done 
-}
+STRIDE_EPOCH_EPOCH_DURATION="35s"

integration-tests/network/scripts/constants.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+SCRIPTS_DIR=scripts
+CONFIG_DIR=configs
+
+VALIDATOR_KEYS_DIR=validator-keys
+NODE_KEYS_DIR=node-keys
+NODE_IDS_DIR=node-ids
+GENESIS_DIR=genesis
+KEYS_FILE=${CONFIG_DIR}/keys.json
+
+PEER_PORT=26656
+RPC_PORT=26657
+
+API_ENDPOINT=${API_ENDPOINT:-'http://api.integration.svc:8000'}