[LocalNet] Replace AppGateServer with Path in LocalNet (#879)

## Summary Add PATH to Tilt so it could run on LocalNet for E2E tests and local load testing. ## Issue In order to deprecate AppGate server and have better integration with the production gateway component, Path needs to run on LocalNet for both E2E test and load testing. - #838 ## Type of change - [x] New feature, functionality or library ## Testing - [x] **LocalNet E2E Tests**: `make test_e2e` - [ ] **DevNet E2E Tests**: Add the `devnet-test-e2e` label to the PR. ## Sanity Checklist - [x] I have tested my changes using the available tooling - [ ] I have commented my code - [x] I have performed a self-review of my own code; both comments & source code - [ ] I create and reference any new tickets, if applicable - [ ] I have left TODOs throughout the codebase, if applicable --------- Co-authored-by: Daniel Olshansky <[email protected]> Co-authored-by: Dmitry K <[email protected]> Co-authored-by: Dima K. <[email protected]>
pokt-network · Dec 10, 2024 · 4000144 · 4000144
1 parent 2743cbf
commit 4000144
Show file tree

Hide file tree

Showing 45 changed files with 291 additions and 2,459 deletions.
diff --git a/.env.dev b/.env.dev
@@ -7,13 +7,12 @@ POKTROLLD_HOME=./localnet/poktrolld
 POCKET_NODE=tcp://127.0.0.1:26657
 # TestNet RPC endpoint for validator maintained by Grove. Needs to be updated if there's another "primary" testnet.
 TESTNET_RPC=https://testnet-validated-validator-rpc.poktroll.com/
-APPGATE_SERVER=http://localhost:42069
-GATEWAY_URL=http://localhost:42079
+PATH_URL=http://localhost:3000
 POCKET_ADDR_PREFIX=pokt
 CHAIN_ID=poktroll
 
 # The domain ending in ".town" is staging, ".city" is production
-GROVE_GATEWAY_STAGING_ETH_MAINNET=https://eth-mainnet.rpc.grove.town
+GROVE_PORTAL_STAGING_ETH_MAINNET=https://eth-mainnet.rpc.grove.town
 # The "protocol" field here instructs the Grove gateway which network to use
 JSON_RPC_DATA_ETH_BLOCK_HEIGHT='{"protocol": "shannon-testnet","jsonrpc":"2.0","id":"0","method":"eth_blockNumber", "params": []}'
 

diff --git a/.github/workflows-helpers/run-e2e-test-job-template.yaml b/.github/workflows-helpers/run-e2e-test-job-template.yaml
@@ -11,8 +11,8 @@ spec:
         pokt.network/purpose: e2e-tests
     spec:
       initContainers:
-        # We need to make sure the services like validator, relayminer, appgate server, etc. work before we execute
-        # an e2e test.
+        # We need to ensure services like Validator, RelayMiner, PATH, etc. work
+        # before executing an e2e test.
         - name: check-services
           image: ruby:3.0
           command: ["ruby"]
@@ -49,10 +49,14 @@ spec:
               value: "false" # Flip to true to see the command and result of the execution
             - name: POKTROLLD_HOME
               value: /root/.poktroll
-            - name: APPGATE_SERVER_URL
-              value: http://${NAMESPACE}-appgate-server:80
-            - name: GATEWAY_URL
-              value: http://${NAMESPACE}-gateway:80
+            - name: PATH_URL
+              value: http://${NAMESPACE}-path:3000/v1
+              # PATH relies on subdomains to get the requested service but our DevNet infra is not
+              # built to expose arbitrary subdomains and supporting it would be a significant effort.
+              # As a workaround, PATH_HOST_OVERRIDE is used as the host:port to connect to PATH while
+              # the subdomain is passed as a Host header in the request.
+            - name: PATH_HOST_OVERRIDE
+              value: ${NAMESPACE}-path:3000
           volumeMounts:
             - mountPath: /root/.poktroll/keyring-test/
               name: writable-keys-volume

diff --git a/Makefile b/Makefile
@@ -5,13 +5,12 @@ SHELL = /bin/sh
 POKTROLLD_HOME ?= ./localnet/poktrolld
 POCKET_NODE ?= tcp://127.0.0.1:26657 # The pocket node (validator in the localnet context)
 TESTNET_RPC ?= https://testnet-validated-validator-rpc.poktroll.com/ # TestNet RPC endpoint for validator maintained by Grove. Needs to be update if there's another "primary" testnet.
-APPGATE_SERVER ?= http://localhost:42069
-GATEWAY_URL ?= http://localhost:42079
+PATH_URL ?= http://localhost:3000
 POCKET_ADDR_PREFIX = pokt
 LOAD_TEST_CUSTOM_MANIFEST ?= loadtest_manifest_example.yaml
 
 # The domain ending in ".town" is staging, ".city" is production
-GROVE_GATEWAY_STAGING_ETH_MAINNET = https://eth-mainnet.rpc.grove.town
+GROVE_PORTAL_STAGING_ETH_MAINNET = https://eth-mainnet.rpc.grove.town
 # JSON RPC data for a test relay request
 JSON_RPC_DATA_ETH_BLOCK_HEIGHT = '{"jsonrpc":"2.0","id":"0","method":"eth_blockNumber", "params": []}'
 
@@ -349,14 +348,19 @@ docusaurus_start: check_npm check_node ## Start the Docusaurus server
 docs_update_gov_params_page: ## Update the page in Docusaurus documenting all the governance parameters
 	go run tools/scripts/docusaurus/generate_docs_params.go
 
-######################
-### Ignite Helpers ###
-######################
+#######################
+### Keyring Helpers ###
+#######################
+
 
 .PHONY: poktrolld_addr
 poktrolld_addr: ## Retrieve the address for an account by ACC_NAME
 	@echo $(shell poktrolld --home=$(POKTROLLD_HOME) keys show -a $(ACC_NAME))
 
+.PHONY: poktrolld_key
+poktrolld_key: ## Retrieve the private key for an account by ACC_NAME
+	@echo $(shell poktrolld --home=$(POKTROLLD_HOME) keys export --unsafe --unarmored-hex $(ACC_NAME))
+
 ###################
 ### Act Helpers ###
 ###################
@@ -415,13 +419,13 @@ release_tag_minor_release: ## Tag a new minor release (e.g. v1.0.0 -> v1.1.0)
 	@echo "  git push origin $(NEW_TAG)"
 	@echo "And draft a new release at https://github.com/pokt-network/poktroll/releases/new"
 
-#############################
-### Grove Gateway Helpers ###
-#############################
+############################
+### Grove Portal Helpers ###
+############################
 
 .PHONY: grove_staging_eth_block_height
 grove_staging_eth_block_height: ## Sends a relay through the staging grove gateway to the eth-mainnet chain. Must have GROVE_STAGING_PORTAL_APP_ID environment variable set.
-	curl $(GROVE_GATEWAY_STAGING_ETH_MAINNET)/v1/$(GROVE_STAGING_PORTAL_APP_ID) \
+	curl $(GROVE_PORTAL_STAGING_ETH_MAINNET)/v1/$(GROVE_STAGING_PORTAL_APP_ID) \
 		-H 'Content-Type: application/json' \
 		-H 'Protocol: shannon-testnet' \
 		--data $(JSON_RPC_DATA_ETH_BLOCK_HEIGHT)
@@ -437,6 +441,7 @@ grove_staging_eth_block_height: ## Sends a relay through the staging grove gatew
 ###############
 ### Imports ###
 ###############
+
 include ./makefiles/warnings.mk
 include ./makefiles/todos.mk
 include ./makefiles/checks.mk
@@ -450,3 +455,4 @@ include ./makefiles/suppliers.mk
 include ./makefiles/gateways.mk
 include ./makefiles/session.mk
 include ./makefiles/claims.mk
+include ./makefiles/relay.mk
diff --git a/Tiltfile b/Tiltfile
@@ -46,41 +46,41 @@ localnet_config_defaults = {
             "level": "debug",
         },
     },
-    "gateways": {
-        "count": 1,
-        "delve": {"enabled": False},
-        "logs": {
-            "level": "debug",
-        },
-    },
-    "appgateservers": {
-        "count": 1,
-        "delve": {"enabled": False},
-        "logs": {
-            "level": "debug",
-        },
-    },
     "ollama": {
         "enabled": False,
         "model": "qwen:0.5b",
     },
     "rest": {
         "enabled": True,
     },
+    "path_gateways": {
+        "count": 1,
+    },
 
-    # NOTE: git submodule usage was explicitly avoided to reduce environment complexity.
+    #############
+    # NOTE: git submodule usage was explicitly avoided for the repositories below
+    # to reduce environment complexity.
+    #############
 
     # By default, we use the `helm_repo` function below to point to the remote repository
     # but can update it to the locally cloned repo for testing & development
     "helm_chart_local_repo": {
         "enabled": False,
         "path": os.path.join("..", "helm-charts")
     },
+
+    # By default, we use a pre-built PATH image, but can update it to use a local
+    # repo instead.
+    "path_local_repo": {
+        "enabled": False,
+        "path": "../path"
+    },
+
     "indexer": {
         "repo_path": os.path.join("..", "pocketdex"),
         "enabled": True,
         "clone_if_not_present": False,
-    }
+    },
 }
 localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults)
 # Initial empty config
@@ -96,9 +96,12 @@ if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_con
     print("Updating " + localnet_config_path + " with defaults")
     local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config))
 
-# Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
+# Configure helm chart reference.
+# If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
 helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/")
-# TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
+helm_repo("buildwithgrove", "https://buildwithgrove.github.io/helm-charts/")
+
+# Configure POKT chart references
 chart_prefix = "pokt-network/"
 if localnet_config["helm_chart_local_repo"]["enabled"]:
     helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"]
@@ -107,6 +110,15 @@ if localnet_config["helm_chart_local_repo"]["enabled"]:
     # TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
     chart_prefix = helm_chart_local_repo + "/charts/"
 
+# Configure PATH references
+grove_chart_prefix = "buildwithgrove/"
+# If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
+path_local_repo = ""
+if localnet_config["path_local_repo"]["enabled"]:
+    path_local_repo = localnet_config["path_local_repo"]["path"]
+    hot_reload_dirs.append(path_local_repo)
+    print("Using local PATH repo " + path_local_repo)
+
 # Observability
 print("Observability enabled: " + str(localnet_config["observability"]["enabled"]))
 if localnet_config["observability"]["enabled"]:
@@ -290,91 +302,70 @@ for x in range(localnet_config["relayminers"]["count"]):
         ],
     )
 
-# Provision AppGate Servers
+if localnet_config["path_local_repo"]["enabled"]:
+    docker_build("path-local", path_local_repo)
+
+# TODO_MAINNET(@okdas): Find and replace all `appgateserver` in ./localnet/grafana-dashboards`
+# with PATH metrics (see the .json files)
+# Ref: https://github.com/buildwithgrove/path/pull/72
+
+# Provision PATH Gateway(s)
 actor_number = 0
-for x in range(localnet_config["appgateservers"]["count"]):
-    actor_number = actor_number + 1
-    helm_resource(
-        "appgateserver" + str(actor_number),
-        chart_prefix + "appgate-server",
-        flags=[
-            "--values=./localnet/kubernetes/values-common.yaml",
-            "--values=./localnet/kubernetes/values-appgateserver.yaml",
-            "--set=config.signing_key=app" + str(actor_number),
-            "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
-            "--set=development.delve.enabled=" + str(localnet_config["appgateservers"]["delve"]["enabled"]),
-            "--set=logLevel=" + str(localnet_config["appgateservers"]["logs"]["level"]),
-            "--set=image.repository=poktrolld",
-        ],
-        image_deps=["poktrolld"],
-        image_keys=[("image.repository", "image.tag")],
-    )
-    k8s_resource(
-        "appgateserver" + str(actor_number),
-        labels=["gateways"],
-        resource_deps=["validator"],
-        links=[
-            link(
-                "http://localhost:3003/d/appgateserver/protocol-appgate-server?orgId=1&refresh=5s&var-appgateserver=appgateserver"
-                + str(actor_number),
-                "Grafana dashboard",
-            ),
-        ],
-        port_forwards=[
-            str(42068 + actor_number) + ":42069",  # appgateserver1 - exposes 42069, appgateserver2 exposes 42070, etc.
-            str(40054 + actor_number)
-            + ":40004",  # DLV port. appgateserver1 - exposes 40055, appgateserver2 exposes 40056, etc.
-            # Run `curl localhost:PORT` to see the current snapshot of appgateserver metrics.
-            str(9079 + actor_number)
-            + ":9090",  # appgateserver metrics port. appgateserver1 - exposes 9080, appgateserver2 exposes 9081, etc.
-            # Use with pprof like this: `go tool pprof -http=:3333 http://localhost:6080/debug/pprof/goroutine`
-            str(6079 + actor_number)
-            + ":6090",  # appgateserver metrics port. appgateserver1 - exposes 6080, appgateserver2 exposes 6081, etc.
-        ],
+# Loop to configure and apply multiple PATH gateway deployments
+for x in range(localnet_config["path_gateways"]["count"]):
+    actor_number += 1
+
+    resource_flags = [
+        "--values=./localnet/kubernetes/values-common.yaml",
+        "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
+        "--set=path.mountConfigMaps[0].name=path-config-" + str(actor_number),
+        "--set=path.mountConfigMaps[0].mountPath=/app/config/",
+    ]
+
+    if localnet_config["path_local_repo"]["enabled"]:
+        path_image_deps = ["path-local"]
+        path_image_keys = [("image.repository", "image.tag")]
+        path_deps=["path-local"]
+        resource_flags.append("--set=global.imagePullPolicy=Never")
+    else:
+        path_image_deps = []
+        path_image_keys = []
+        path_deps=[]
+
+    configmap_create(
+        "path-config-" + str(actor_number),
+        from_file=".config.yaml=./localnet/kubernetes/config-path-" + str(actor_number) + ".yaml"
     )
 
-# Provision Gateways
-actor_number = 0
-for x in range(localnet_config["gateways"]["count"]):
-    actor_number = actor_number + 1
     helm_resource(
-        "gateway" + str(actor_number),
-        chart_prefix + "appgate-server",
-        flags=[
-            "--values=./localnet/kubernetes/values-common.yaml",
-            "--values=./localnet/kubernetes/values-gateway.yaml",
-            "--set=config.signing_key=gateway" + str(actor_number),
-            "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
-            "--set=development.delve.enabled=" + str(localnet_config["gateways"]["delve"]["enabled"]),
-            "--set=logLevel=" + str(localnet_config["gateways"]["logs"]["level"]),
-            "--set=image.repository=poktrolld",
-        ],
-        image_deps=["poktrolld"],
-        image_keys=[("image.repository", "image.tag")],
+        "path" + str(actor_number),
+        grove_chart_prefix + "path",
+        flags=resource_flags,
+        image_deps=path_image_deps,
+        image_keys=path_image_keys,
     )
+
+    # Apply the deployment to Kubernetes using Tilt
     k8s_resource(
-        "gateway" + str(actor_number),
+        "path" + str(actor_number),
         labels=["gateways"],
-        resource_deps=["validator"],
-        links=[
-            link(
-                "http://localhost:3003/d/appgateserver/protocol-appgate-server?orgId=1&refresh=5s&var-appgateserver=gateway"
-                + str(actor_number),
-                "Grafana dashboard",
-            ),
-        ],
+        resource_deps=path_deps,
+        # TODO_IMPROVE(@okdas): Update this once PATH has grafana dashboards
+        # links=[
+        #     link(
+        #         "http://localhost:3003/d/path/protocol-path?orgId=1&refresh=5s&var-path=gateway"
+        #         + str(actor_number),
+        #         "Grafana dashboard",
+        #     ),
+        # ],
+        # TODO_IMPROVE(@okdas): Add port forwards to grafana, pprof, like the other resources
         port_forwards=[
-            str(42078 + actor_number) + ":42069",  # gateway1 - exposes 42079, gateway2 exposes 42080, etc.
-            str(40064 + actor_number) + ":40004",  # DLV port. gateway1 - exposes 40065, gateway2 exposes 40066, etc.
-            # Run `curl localhost:PORT` to see the current snapshot of gateway metrics.
-            str(9059 + actor_number)
-            + ":9090",  # gateway metrics port. gateway1 - exposes 9060, gateway2 exposes 9061, etc.
-            # Use with pprof like this: `go tool pprof -http=:3333 http://localhost:6090/debug/pprof/goroutine`
-            str(6059 + actor_number)
-            + ":6060",  # gateway metrics port. gateway1 - exposes 6060, gateway2 exposes 6061, etc.
+                str(2999 + actor_number) + ":3000"
         ],
     )
 
+
+# Provision Validators
 k8s_resource(
     "validator",
     labels=["pocket_network"],
@@ -393,8 +384,10 @@ k8s_resource(
     ],
 )
 
+# Provision anvil (test Ethereum) service nodes
 k8s_resource("anvil", labels=["data_nodes"], port_forwards=["8547"])
 
+# Provision ollama (LLM) service nodes
 if localnet_config["ollama"]["enabled"]:
     print("Ollama enabled: " + str(localnet_config["ollama"]["enabled"]))
 
@@ -413,6 +406,7 @@ if localnet_config["ollama"]["enabled"]:
         resource_deps=["ollama"],
     )
 
+# Provision RESTful (not JSON-RPC) test service nodes
 if localnet_config["rest"]["enabled"]:
     print("REST enabled: " + str(localnet_config["rest"]["enabled"]))
     deployment_create("rest", image="davarski/go-rest-api-demo")

diff --git a/cmd/poktrolld/cmd/root.go b/cmd/poktrolld/cmd/root.go
@@ -26,7 +26,6 @@ import (
 	"github.com/spf13/pflag"
 
 	"github.com/pokt-network/poktroll/app"
-	appgateservercmd "github.com/pokt-network/poktroll/pkg/appgateserver/cmd"
 	relayercmd "github.com/pokt-network/poktroll/pkg/relayer/cmd"
 )
 
@@ -135,10 +134,9 @@ func NewRootCmd() *cobra.Command {
 		relayercmd.RelayerCmd(),
 	)
 
-	// add the appgate server command
-	rootCmd.AddCommand(
-		appgateservercmd.AppGateServerCmd(),
-	)
+	// TODO_MAINNET(@commoddity): Consider adding an entrypoint to deploy a PATH
+	// gateway to the localnet to streamline it for users.
+	// For reference, see how we removed appgateserver in #879.
 
 	return rootCmd
 }