diff --git a/.github/label-actions.yml b/.github/label-actions.yml index 0ab29008d..8618e0d5a 100644 --- a/.github/label-actions.yml +++ b/.github/label-actions.yml @@ -1,13 +1,13 @@ -# When `devnet-e2e-test` is added, also assign `devnet` to the PR. -devnet-e2e-test: +# When `devnet-test-e2e` is added, also assign `devnet` to the PR. +devnet-test-e2e: prs: comment: The CI will now also run the e2e tests on devnet, which increases the time it takes to complete all CI checks. label: - devnet - push-image -# When `devnet-e2e-test` is removed, also delete `devnet` from the PR. --devnet-e2e-test: +# When `devnet-test-e2e` is removed, also delete `devnet` from the PR. +-devnet-test-e2e: prs: unlabel: - devnet @@ -18,11 +18,11 @@ devnet: label: - push-image -# When `devnet` is removed, also delete `devnet-e2e-test` from the PR. +# When `devnet` is removed, also delete `devnet-test-e2e` from the PR. -devnet: prs: unlabel: - - devnet-e2e-test + - devnet-test-e2e # Let the developer know that they need to push another commit after attaching the label to PR. push-image: @@ -34,4 +34,4 @@ push-image: prs: unlabel: - devnet - - devnet-e2e-test + - devnet-test-e2e diff --git a/.github/workflows-helpers/run-e2e-test-job-template.yaml b/.github/workflows-helpers/run-e2e-test-job-template.yaml new file mode 100644 index 000000000..6ad0def13 --- /dev/null +++ b/.github/workflows-helpers/run-e2e-test-job-template.yaml @@ -0,0 +1,45 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: ${NAMESPACE} +spec: + ttlSecondsAfterFinished: 120 + template: + spec: + containers: + - name: e2e-tests + image: ghcr.io/pokt-network/poktrolld:${IMAGE_TAG} + command: ["/bin/sh"] + args: ["-c", "poktrolld q gateway list-gateway --node=$POCKET_NODE && poktrolld q application list-application --node=$POCKET_NODE && poktrolld q supplier list-supplier --node=$POCKET_NODE && go test -v ./e2e/tests/... -tags=e2e"] + env: + - name: AUTH_TOKEN + valueFrom: + secretKeyRef: + key: auth_token + name: celestia-secret + - name: POCKET_NODE + value: tcp://${NAMESPACE}-sequencer:36657 + - name: E2E_DEBUG_OUTPUT + value: "false" # Flip to true to see the command and result of the execution + - name: POKTROLLD_HOME + value: /root/.pocket + - name: CELESTIA_HOSTNAME + value: celestia-rollkit + volumeMounts: + - mountPath: /root/.pocket/keyring-test/ + name: keys-volume + - mountPath: /root/.pocket/config/ + name: configs-volume + restartPolicy: Never + volumes: + - configMap: + defaultMode: 420 + name: poktrolld-keys + name: keys-volume + - configMap: + defaultMode: 420 + name: poktrolld-configs + name: configs-volume + serviceAccountName: default + backoffLimit: 0 diff --git a/.github/workflows-helpers/run-e2e-test.sh b/.github/workflows-helpers/run-e2e-test.sh new file mode 100644 index 000000000..723ab1eca --- /dev/null +++ b/.github/workflows-helpers/run-e2e-test.sh @@ -0,0 +1,55 @@ +# Check if the pod with the matching image SHA and purpose is ready +echo "Checking for ready sequencer pod with image SHA ${IMAGE_TAG}..." +while : ; do +# Get all pods with the matching purpose +PODS_JSON=$(kubectl get pods -n ${NAMESPACE} -l pokt.network/purpose=sequencer -o json) + +# Check if any pods are running and have the correct image SHA +READY_POD=$(echo $PODS_JSON | jq -r ".items[] | select(.status.phase == \"Running\") | select(.spec.containers[].image | contains(\"${IMAGE_TAG}\")) | .metadata.name") + +if [[ -n "${READY_POD}" ]]; then + echo "Ready pod found: ${READY_POD}" + break +else + echo "Sequencer with with an image ${IMAGE_TAG} is not ready yet. Will retry in 10 seconds..." + sleep 10 +fi +done + +# Create a job to run the e2e tests +envsubst < .github/workflows-helpers/run-e2e-test-job-template.yaml > job.yaml +kubectl apply -f job.yaml + +# Wait for the pod to be created and be in a running state +echo "Waiting for the pod to be in the running state..." +while : ; do +POD_NAME=$(kubectl get pods -n ${NAMESPACE} --selector=job-name=${JOB_NAME} -o jsonpath='{.items[*].metadata.name}') +[[ -z "${POD_NAME}" ]] && echo "Waiting for pod to be scheduled..." && sleep 5 && continue +POD_STATUS=$(kubectl get pod ${POD_NAME} -n ${NAMESPACE} -o jsonpath='{.status.phase}') +[[ "${POD_STATUS}" == "Running" ]] && break +echo "Current pod status: ${POD_STATUS}" +sleep 5 +done + +echo "Pod is running. Monitoring logs and status..." +# Stream the pod logs in the background +kubectl logs -f ${POD_NAME} -n ${NAMESPACE} & + +# Monitor pod status in a loop +while : ; do +CURRENT_STATUS=$(kubectl get pod ${POD_NAME} -n ${NAMESPACE} -o jsonpath="{.status.containerStatuses[0].state}") +if echo $CURRENT_STATUS | grep -q 'terminated'; then + EXIT_CODE=$(echo $CURRENT_STATUS | jq '.terminated.exitCode') + if [[ "$EXIT_CODE" != "0" ]]; then + echo "Container terminated with exit code ${EXIT_CODE}" + kubectl delete job ${JOB_NAME} -n ${NAMESPACE} + exit 1 + fi + break +fi +sleep 5 +done + +# If the loop exits without failure, the job succeeded +echo "Job completed successfully" +kubectl delete job ${JOB_NAME} -n ${NAMESPACE} diff --git a/.github/workflows/go.yml b/.github/workflows/main-build.yml similarity index 63% rename from .github/workflows/go.yml rename to .github/workflows/main-build.yml index 6f76334c1..83893df60 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/main-build.yml @@ -1,7 +1,4 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Ignite build & test +name: Main build on: push: @@ -9,11 +6,11 @@ on: pull_request: concurrency: - group: ${{ github.head_ref || github.ref_name }} + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true jobs: - build: + build-push-container: runs-on: ubuntu-latest steps: - name: install ignite @@ -29,7 +26,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "1.20" + go-version: "1.20.10" - name: Install CI dependencies run: make install_ci_deps @@ -37,18 +34,9 @@ jobs: - name: Generate protobufs run: make proto_regen - - name: Generate mocks - run: make go_mockgen - - - name: Run golangci-lint - run: make go_lint - - name: Build run: ignite chain build -v --debug --skip-proto - - name: Test - run: make go_test - - name: Set up Docker Buildx if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) uses: docker/setup-buildx-action@v3 @@ -61,7 +49,7 @@ jobs: DOCKER_METADATA_PR_HEAD_SHA: "true" with: images: | - ghcr.io/pokt-network/pocketd + ghcr.io/pokt-network/poktrolld tags: | type=ref,event=branch type=ref,event=pr @@ -76,11 +64,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Copy binary to inside of the Docker context + - name: Copy binaries to inside of the Docker context if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) run: | mkdir -p ./bin # Make sure the bin directory exists + cp $(which ignite) ./bin # Copy ignite binary to the repo's bin directory cp $(go env GOPATH)/bin/poktrolld ./bin # Copy the binary to the repo's bin directory + ls -la ./bin - name: Build and push Docker image if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) @@ -95,3 +85,36 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . + + run-e2e-tests: + needs: build-push-container + if: contains(github.event.pull_request.labels.*.name, 'devnet-test-e2e') + runs-on: ubuntu-latest + env: + GKE_CLUSTER: protocol-us-central1 + GKE_ZONE: us-central1 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + .github + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GKE_PROTOCOL_US_CENTRAL }}' + + - uses: google-github-actions/get-gke-credentials@v1 + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + project_id: ${{ secrets.GKE_PROTOCOL_PROJECT }} + + - name: Run E2E test job + env: + IMAGE_TAG: sha-${{ github.event.pull_request.head.sha || github.sha }} + NAMESPACE: devnet-issue-${{ github.event.number }} + JOB_NAME: e2e-test-${{ github.event.pull_request.head.sha || github.sha }} + POCKET_NODE: tcp://devnet-issue-${{ github.event.number }}-sequencer:36657 + run: bash .github/workflows-helpers/run-e2e-test.sh \ No newline at end of file diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 62afe323e..2fb9c9dd7 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -4,6 +4,10 @@ on: pull_request: branches: ["main"] +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + jobs: # Makes sure that comments like TODO_IN_THIS_PR or TODO_IN_THIS_COMMIT block merging to main # More info: https://github.com/pokt-network/action-fail-on-found diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 000000000..6bd5990c7 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,48 @@ +name: Run tests + +on: + push: + branches: ["main"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + GKE_CLUSTER: protocol-us-central1 + GKE_ZONE: us-central1 + +jobs: + go-test: + runs-on: ubuntu-latest + steps: + - name: install ignite + # If this step fails due to ignite.com failing, see #116 for a temporary workaround + run: | + curl https://get.ignite.com/cli! | bash + ignite version + + - uses: actions/checkout@v3 + with: + fetch-depth: "0" # Per https://github.com/ignite/cli/issues/1674#issuecomment-1144619147 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.20.10" + + - name: Install CI dependencies + run: make install_ci_deps + + - name: Generate protobufs + run: make proto_regen + + - name: Generate mocks + run: make go_mockgen + + - name: Run golangci-lint + run: make go_lint + + - name: Test + run: make go_test diff --git a/.gitignore b/.gitignore index b6ff1eab7..8fe75f2aa 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,6 @@ localnet/*/config/*.json !localnet/poktrolld/config/client.toml !localnet/poktrolld/config/config.toml - # Macos .DS_Store **/.DS_Store @@ -63,12 +62,11 @@ localnet_config.yaml release # SMT KVStore files +# TODO_TECHDEBT(#126, @red-0ne): Rename `smt` to `smt_stores` and make it configurable so it can be stored anywhere on this smt # Do not allow a multi-moduled projected go.work.sum -# TODO_IN_THIS_COMMIT: Why did we start generating .dot files? +# TODO_TECHDEBT: It seems that .dot files come and go so we need to figure out the root cause: https://github.com/pokt-network/poktroll/pull/177/files#r1392521547 # **/*.dot - - diff --git a/Dockerfile.dev b/Dockerfile.dev index 2d10955e0..0c4fe64df 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -6,7 +6,7 @@ FROM golang:1.20 as base RUN apt update && \ apt-get install -y \ ca-certificates \ - curl jq make + curl jq make vim less # enable faster module downloading. ENV GOPROXY https://proxy.golang.org @@ -15,9 +15,12 @@ COPY . /poktroll WORKDIR /poktroll -RUN mv /poktroll/bin/poktrolld /usr/bin/poktrolld +RUN mv /poktroll/bin/ignite /usr/bin/ && mv /poktroll/bin/poktrolld /usr/bin/ +# TODO_TECHDEBT(@okdas): Ports are not documented as they will soon be changed with a document to follow EXPOSE 8545 EXPOSE 8546 +EXPOSE 8547 +EXPOSE 8548 ENTRYPOINT ["ignite"] diff --git a/Makefile b/Makefile index 34ffc11c7..19120c578 100644 --- a/Makefile +++ b/Makefile @@ -282,6 +282,7 @@ app_list: ## List all the staked applications app_stake: ## Stake tokens for the application specified (must specify the APP and SERVICES env vars) poktrolld --home=$(POKTROLLD_HOME) tx application stake-application 1000upokt $(SERVICES) --keyring-backend test --from $(APP) --node $(POCKET_NODE) +# TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper .PHONY: app1_stake app1_stake: ## Stake app1 (also staked in genesis) APP=app1 SERVICES=anvil,svc1,svc2 make app_stake @@ -362,22 +363,22 @@ supplier_list: ## List all the staked supplier supplier_stake: ## Stake tokens for the supplier specified (must specify the APP env var) poktrolld --home=$(POKTROLLD_HOME) tx supplier stake-supplier 1000upokt "$(SERVICES)" --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) +# TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper .PHONY: supplier1_stake supplier1_stake: ## Stake supplier1 (also staked in genesis) - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). - # TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8545,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake supplier2_stake: ## Stake supplier2 - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8545,svc2;http://localhost:8082" make supplier_stake .PHONY: supplier3_stake supplier3_stake: ## Stake supplier3 - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8545,svc3;http://localhost:8083" make supplier_stake diff --git a/Tiltfile b/Tiltfile index 1333a406e..95e8c9a6b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,13 +2,14 @@ load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_resource", "helm_repo") # A list of directories where changes trigger a hot-reload of the sequencer -hot_reload_dirs = ["app", "cmd", "tools", "x"] +hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"] # Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values localnet_config_path = "localnet_config.yaml" localnet_config_defaults = { - "relayers": {"count": 1}, + "relayminers": {"count": 1}, "gateways": {"count": 1}, + "appgateservers": {"count": 1}, # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, @@ -25,16 +26,12 @@ if (localnet_config_file != localnet_config) or ( # Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo. helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/") -sequencer_chart = "pokt-network/poktroll-sequencer" -poktroll_chart = "pokt-network/poktroll" +chart_prefix = "pokt-network/" if localnet_config["helm_chart_local_repo"]["enabled"]: helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"] hot_reload_dirs.append(helm_chart_local_repo) print("Using local helm chart repo " + helm_chart_local_repo) - - sequencer_chart = helm_chart_local_repo + "/charts/poktroll-sequencer" - poktroll_chart = helm_chart_local_repo + "/charts/poktroll" - + chart_prefix = helm_chart_local_repo + "/charts/" # Import files into Kubernetes ConfigMap def read_files_from_directory(directory): @@ -114,20 +111,32 @@ k8s_yaml( ["localnet/kubernetes/celestia-rollkit.yaml", "localnet/kubernetes/anvil.yaml"] ) -# Run pocket-specific nodes (sequencer, relayers, etc...) +# Run pocket-specific nodes (sequencer, relayminers, etc...) helm_resource( "sequencer", - sequencer_chart, + chart_prefix + "poktroll-sequencer", flags=["--values=./localnet/kubernetes/values-common.yaml"], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], ) helm_resource( - "relayers", - poktroll_chart, + "relayminers", + chart_prefix + "relayminer", flags=[ "--values=./localnet/kubernetes/values-common.yaml", - "--set=replicaCount=" + str(localnet_config["relayers"]["count"]), + "--values=./localnet/kubernetes/values-relayminer.yaml", + "--set=replicaCount=" + str(localnet_config["relayminers"]["count"]), + ], + image_deps=["poktrolld"], + image_keys=[("image.repository", "image.tag")], +) +helm_resource( + "appgateservers", + chart_prefix + "appgate-server", + flags=[ + "--values=./localnet/kubernetes/values-common.yaml", + "--values=./localnet/kubernetes/values-appgateserver.yaml", + "--set=replicaCount=" + str(localnet_config["appgateservers"]["count"]), ], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], @@ -146,9 +155,15 @@ k8s_resource( port_forwards=["36657", "40004"], ) k8s_resource( - "relayers", + "relayminers", + labels=["blockchains"], + resource_deps=["sequencer"], + port_forwards=["8548", "40005"], +) +k8s_resource( + "appgateservers", labels=["blockchains"], resource_deps=["sequencer"], - port_forwards=["8545", "8546", "40005"], + port_forwards=["42069", "40006"], ) k8s_resource("anvil", labels=["blockchains"], port_forwards=["8547"]) diff --git a/config.yml b/config.yml index 3d8094665..4dac5f556 100644 --- a/config.yml +++ b/config.yml @@ -100,7 +100,7 @@ genesis: - endpoints: - configs: [] rpc_type: JSON_RPC - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to it instead of `localhost`. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to it instead of `localhost`. url: http://localhost:8545 service: id: anvil diff --git a/e2e/tests/node.go b/e2e/tests/node.go index afd42a8c8..440313431 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -23,6 +23,8 @@ var ( defaultHome = os.Getenv("POKTROLLD_HOME") // defaultAppGateServerURL used by curl commands to send relay requests defaultAppGateServerURL = os.Getenv("APPGATE_SERVER") + // defaultDebugOutput provides verbose output on manipulations with binaries (cli command, stdout, stderr) + defaultDebugOutput = os.Getenv("E2E_DEBUG_OUTPUT") ) func init() { @@ -104,6 +106,10 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { err = fmt.Errorf("error running command [%s]: %v, stderr: %s", commandStr, err, stderrBuf.String()) } + if defaultDebugOutput == "true" { + fmt.Printf("%#v\n", r) + } + return r, err } diff --git a/localnet/kubernetes/values-appgateserver.yaml b/localnet/kubernetes/values-appgateserver.yaml new file mode 100644 index 000000000..cefc1887d --- /dev/null +++ b/localnet/kubernetes/values-appgateserver.yaml @@ -0,0 +1,2 @@ +pocket: + node: sequencer-poktroll-sequencer \ No newline at end of file diff --git a/localnet/kubernetes/values-relayminer.yaml b/localnet/kubernetes/values-relayminer.yaml new file mode 100644 index 000000000..cefc1887d --- /dev/null +++ b/localnet/kubernetes/values-relayminer.yaml @@ -0,0 +1,2 @@ +pocket: + node: sequencer-poktroll-sequencer \ No newline at end of file diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 47eba4e40..7a541a511 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -7,24 +7,25 @@ import ( "log" "net/http" "net/url" - "os" - "os/signal" "cosmossdk.io/depinject" cosmosclient "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/client/flags" + cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" "github.com/spf13/cobra" + "github.com/pokt-network/poktroll/cmd/signals" "github.com/pokt-network/poktroll/pkg/appgateserver" - blockclient "github.com/pokt-network/poktroll/pkg/client/block" - eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" + "github.com/pokt-network/poktroll/pkg/deps/config" ) +// We're `explicitly omitting default` so that the appgateserver crashes if these aren't specified. +const omittedDefaultFlagValue = "explicitly omitting default" + var ( flagSigningKey string flagSelfSigning bool flagListeningEndpoint string - flagCometWebsocketUrl string + flagQueryNodeUrl string ) func AppGateServerCmd() *cobra.Command { @@ -56,13 +57,15 @@ relays to the AppGate server and function as an Application, provided that: RunE: runAppGateServer, } + // Custom flags cmd.Flags().StringVar(&flagSigningKey, "signing-key", "", "The name of the key that will be used to sign relays") cmd.Flags().StringVar(&flagListeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the appgate server will listen on") - cmd.Flags().StringVar(&flagCometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the comet websocket endpoint to communicate with the pocket blockchain") cmd.Flags().BoolVar(&flagSelfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") + cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "tcp://: to a full pocket node for reading data and listening for on-chain events") - cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "The URL of the comet tcp endpoint to communicate with the pocket blockchain") + // Cosmos flags + cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly and uses flagQueryNodeUrl underneath") return cmd } @@ -72,18 +75,8 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { ctx, cancelCtx := context.WithCancel(cmd.Context()) defer cancelCtx() - // Handle interrupts in a goroutine. - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - - // Block until we receive an interrupt or kill signal (OS-agnostic) - <-sigCh - log.Println("INFO: Interrupt signal received, shutting down...") - - // Signal goroutines to stop - cancelCtx() - }() + // Handle interrupt and kill signals asynchronously. + signals.GoOnExitSignal(cancelCtx) // Parse the listening endpoint. listeningUrl, err := url.Parse(flagListeningEndpoint) @@ -92,7 +85,7 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { } // Setup the AppGate server dependencies. - appGateServerDeps, err := setupAppGateServerDependencies(cmd, ctx, flagCometWebsocketUrl) + appGateServerDeps, err := setupAppGateServerDependencies(ctx, cmd) if err != nil { return fmt.Errorf("failed to setup AppGate server dependencies: %w", err) } @@ -127,21 +120,62 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return nil } -func setupAppGateServerDependencies(cmd *cobra.Command, ctx context.Context, cometWebsocketUrl string) (depinject.Config, error) { - // Retrieve the client context for the chain interactions. - clientCtx := cosmosclient.GetClientContextFromCmd(cmd) +func setupAppGateServerDependencies( + ctx context.Context, + cmd *cobra.Command, +) (depinject.Config, error) { + pocketNodeWebsocketUrl, err := getPocketNodeWebsocketUrl() + if err != nil { + return nil, err + } - // Create the events client. - eventsQueryClient := eventsquery.NewEventsQueryClient(cometWebsocketUrl) + supplierFuncs := []config.SupplierFn{ + config.NewSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), + config.NewSupplyBlockClientFn(pocketNodeWebsocketUrl), + newSupplyQueryClientContextFn(flagQueryNodeUrl), + } - // Create the block client. - log.Printf("INFO: Creating block client, using comet websocket URL: %s...", cometWebsocketUrl) - deps := depinject.Supply(eventsQueryClient) - blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) + return config.SupplyConfig(ctx, cmd, supplierFuncs) +} + +// getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to +// connect to for subscribing to on-chain events. +func getPocketNodeWebsocketUrl() (string, error) { + if flagQueryNodeUrl == omittedDefaultFlagValue { + return "", errors.New("missing required flag: --query-node") + } + + pocketNodeURL, err := url.Parse(flagQueryNodeUrl) if err != nil { - return nil, fmt.Errorf("failed to create block client: %w", err) + return "", err } - // Return the dependencies config. - return depinject.Supply(clientCtx, blockClient), nil + return fmt.Sprintf("ws://%s/websocket", pocketNodeURL.Host), nil +} + +// newSupplyQueryClientContextFn returns a new depinject.Config which is supplied with +// the given deps and a new cosmos ClientCtx +func newSupplyQueryClientContextFn(pocketQueryClientUrl string) config.SupplierFn { + return func( + _ context.Context, + deps depinject.Config, + cmd *cobra.Command, + ) (depinject.Config, error) { + // Set --node flag to the pocketQueryClientUrl for the client context + // This flag is read by cosmosclient.GetClientQueryContext. + err := cmd.Flags().Set(cosmosflags.FlagNode, pocketQueryClientUrl) + if err != nil { + return nil, err + } + + // Get the client context from the command. + queryClientCtx, err := cosmosclient.GetClientQueryContext(cmd) + if err != nil { + return nil, err + } + deps = depinject.Configs(deps, depinject.Supply( + queryClientCtx, + )) + return deps, nil + } } diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index a85d5fae3..fe4f15000 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -225,6 +225,8 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ + // TODO_BLOCKER(@red-0ne): This MUST match the Id provided by the request. + // If JSON-RPC request is not unmarshaled yet (i.e. can't extract ID), it SHOULD be a random ID. Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ diff --git a/pkg/deps/config/config.go b/pkg/deps/config/config.go new file mode 100644 index 000000000..9621fa1de --- /dev/null +++ b/pkg/deps/config/config.go @@ -0,0 +1,73 @@ +package config + +import ( + "context" + + "cosmossdk.io/depinject" + "github.com/spf13/cobra" + + "github.com/pokt-network/poktroll/pkg/client/block" + eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" +) + +// SupplierFn is a function that is used to supply a depinject config. +type SupplierFn func( + context.Context, + depinject.Config, + *cobra.Command, +) (depinject.Config, error) + +// SupplyConfig supplies a depinject config by calling each of the supplied +// supplier functions in order and passing the result of each supplier to the +// next supplier, chaining them together. +func SupplyConfig( + ctx context.Context, + cmd *cobra.Command, + suppliers []SupplierFn, +) (deps depinject.Config, err error) { + // Initialize deps to with empty depinject config. + deps = depinject.Configs() + for _, supplyFn := range suppliers { + deps, err = supplyFn(ctx, deps, cmd) + if err != nil { + return nil, err + } + } + return deps, nil +} + +// NewSupplyEventsQueryClientFn constructs an EventsQueryClient instance and returns +// a new depinject.Config which is supplied with the given deps and the new +// EventsQueryClient. +func NewSupplyEventsQueryClientFn( + pocketNodeWebsocketUrl string, +) SupplierFn { + return func( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketUrl) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil + } +} + +// NewSupplyBlockClientFn returns a function with constructs a BlockClient instance +// with the given nodeURL and returns a new +// depinject.Config which is supplied with the given deps and the new +// BlockClient. +func NewSupplyBlockClientFn(pocketNodeWebsocketUrl string) SupplierFn { + return func( + ctx context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + blockClient, err := block.NewBlockClient(ctx, deps, pocketNodeWebsocketUrl) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(blockClient)), nil + } +} diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index a4db8c1ba..4f43c4ff2 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -17,43 +17,41 @@ import ( eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" "github.com/pokt-network/poktroll/pkg/client/supplier" "github.com/pokt-network/poktroll/pkg/client/tx" + "github.com/pokt-network/poktroll/pkg/deps/config" "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/pkg/relayer/miner" "github.com/pokt-network/poktroll/pkg/relayer/proxy" "github.com/pokt-network/poktroll/pkg/relayer/session" ) +// We're `explicitly omitting default` so the relayer crashes if these aren't specified. const omittedDefaultFlagValue = "explicitly omitting default" +// TODO_CONSIDERATION: Consider moving all flags defined in `/pkg` to a `flags.go` file. var ( - flagSigningKeyName string - flagSmtStorePath string - flagSequencerNodeUrl string - flagPocketNodeUrl string + flagSigningKeyName string + flagSmtStorePath string + flagNetworkNodeUrl string + flagQueryNodeUrl string ) -type supplierFn func( - context.Context, - depinject.Config, - *cobra.Command, -) (depinject.Config, error) - func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", Short: "Run a relay miner", - // TODO_TECHDEBT: add a longer long description. Long: `Run a relay miner. The relay miner process configures and starts relay servers for each service the supplier actor identified by --signing-key is -staked for (configured on-chain). Relay requests received by the relay servers -are validated and proxied to their respective service endpoints. The responses +staked for (configured on-chain). + +Relay requests received by the relay servers are validated and proxied to their +respective service endpoints, maintained by the relayer off-chain. The responses are then signed and sent back to the requesting application. For each successfully served relay, the miner will hash and compare its difficulty against an on-chain threshold. If the difficulty is sufficient, it is applicable to relay volume and therefore rewards. Such relays are inserted into and persisted via an SMT KV store. The miner will monitor the current block height and periodically -submit claim and proof messages according to the protocol as sessions become eligable +submit claim and proof messages according to the protocol as sessions become eligible for such operations.`, RunE: runRelayer, } @@ -66,11 +64,9 @@ for such operations.`, // TODO_TECHDEBT(#137): This, alongside other flags, should be part of a config file suppliers provide. cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to where the data backing SMT KV store exists on disk") // Communication flags - // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. - // Figure out what good defaults should be post alpha. - cmd.Flags().StringVar(&flagSequencerNodeUrl, "sequencer-node", "explicitly omitting default", "tcp://: to sequencer node to submit txs") - cmd.Flags().StringVar(&flagPocketNodeUrl, "pocket-node", omittedDefaultFlagValue, "tcp://: to full pocket node for reading data and listening for on-chain events") - cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") + cmd.Flags().StringVar(&flagNetworkNodeUrl, "network-node", omittedDefaultFlagValue, "tcp://: to a pocket node that gossips transactions throughout the network (may or may not be the sequencer") + cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "tcp://: to a full pocket node for reading data and listening for on-chain events") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly and uses flagQueryNodeUrl underneath") return cmd } @@ -120,9 +116,9 @@ func setupRelayerDependencies( return nil, err } - supplierFuncs := []supplierFn{ - newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf - newSupplyBlockClientFn(pocketNodeWebsocketUrl), + supplierFuncs := []config.SupplierFn{ + config.NewSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf + config.NewSupplyBlockClientFn(pocketNodeWebsocketUrl), supplyMiner, // leaf supplyQueryClientContext, // leaf supplyTxClientContext, // leaf @@ -134,26 +130,17 @@ func setupRelayerDependencies( supplyRelayerSessionsManager, } - // Initialize deps to with empty depinject config. - deps = depinject.Configs() - for _, supplyFn := range supplierFuncs { - deps, err = supplyFn(ctx, deps, cmd) - if err != nil { - return nil, err - } - } - - return deps, nil + return config.SupplyConfig(ctx, cmd, supplierFuncs) } // getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to // connect to for subscribing to on-chain events. func getPocketNodeWebsocketUrl() (string, error) { - if flagPocketNodeUrl == omittedDefaultFlagValue { - return "", fmt.Errorf("--pocket-node flag is required") + if flagQueryNodeUrl == omittedDefaultFlagValue { + return "", fmt.Errorf("--query-node flag is required") } - pocketNodeURL, err := url.Parse(flagPocketNodeUrl) + pocketNodeURL, err := url.Parse(flagQueryNodeUrl) if err != nil { return "", err } @@ -166,7 +153,7 @@ func getPocketNodeWebsocketUrl() (string, error) { // EventsQueryClient. func newSupplyEventsQueryClientFn( pocketNodeWebsocketUrl string, -) supplierFn { +) config.SupplierFn { return func( _ context.Context, deps depinject.Config, @@ -182,7 +169,7 @@ func newSupplyEventsQueryClientFn( // with the given nodeURL and returns a new // depinject.Config which is supplied with the given deps and the new // BlockClient. -func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) supplierFn { +func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) config.SupplierFn { return func( ctx context.Context, deps depinject.Config, @@ -212,14 +199,17 @@ func supplyMiner( return depinject.Configs(deps, depinject.Supply(mnr)), nil } +// supplyQueryClientContext returns a function with constructs a ClientContext +// instance with the given cmd and returns a new depinject.Config which is +// supplied with the given deps and the new ClientContext. func supplyQueryClientContext( _ context.Context, deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - // Set --node flag to the --pocket-node for the client context + // Set --node flag to the --query-node for the client context // This flag is read by cosmosclient.GetClientQueryContext. - err := cmd.Flags().Set(cosmosflags.FlagNode, flagPocketNodeUrl) + err := cmd.Flags().Set(cosmosflags.FlagNode, flagQueryNodeUrl) if err != nil { return nil, err } @@ -248,9 +238,9 @@ func supplyTxClientContext( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - // Set --node flag to the --sequencer-node for this client context. + // Set --node flag to the --network-node for this client context. // This flag is read by cosmosclient.GetClientTxContext. - err := cmd.Flags().Set(cosmosflags.FlagNode, flagSequencerNodeUrl) + err := cmd.Flags().Set(cosmosflags.FlagNode, flagNetworkNodeUrl) if err != nil { return nil, err } @@ -354,16 +344,16 @@ func supplyRelayerProxy( deps depinject.Config, _ *cobra.Command, ) (depinject.Config, error) { - // TODO_BLOCKER:(#137): This MUST be populated via the `relayer.json` config file - // TODO_TECHDEBT(#179): this hostname should be updated to match that of the - // in-tilt anvil service. + // TODO_BLOCKER:(#137, @red-0ne): This MUST be populated via the `relayer.json` config file + // TODO_UPNEXT(@okdas): this hostname should be updated to match that of the in-tilt anvil service. proxyServiceURL, err := url.Parse("http://localhost:8547/") if err != nil { return nil, err } - // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented an a local LLM node - // is supported, this needs to be expanded such that a single relayer can proxy to multiple services at once. + // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented AND a local LLM RPC service + // is supported on LocalNet, this needs to be expanded to include more than one service. The ability to support + // multiple services is already in place but currently (as seen below) is hardcoded. proxiedServiceEndpoints := map[string]url.URL{ "anvil": *proxyServiceURL, } @@ -383,6 +373,8 @@ func supplyRelayerProxy( // supplyRelayerSessionsManager constructs a RelayerSessionsManager instance // and returns a new depinject.Config which is supplied with the given deps and // the new RelayerSessionsManager. +// See the comment next to `flagQueryNodeUrl` (if it still exists) on how/why +// we have multiple flags pointing to different node types. func supplyRelayerSessionsManager( ctx context.Context, deps depinject.Config, diff --git a/pkg/relayer/proxy/error_reply.go b/pkg/relayer/proxy/error_reply.go index c6c5606e6..617f3b917 100644 --- a/pkg/relayer/proxy/error_reply.go +++ b/pkg/relayer/proxy/error_reply.go @@ -16,6 +16,8 @@ func (jsrv *jsonRPCServer) replyWithError(writer http.ResponseWriter, err error) relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ + // TODO_BLOCKER(@red-0ne): This MUST match the Id provided by the request. + // If JSON-RPC request is not unmarshaled yet (i.e. can't extract ID), it SHOULD be a random ID. Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 28f92576c..b6c2a5f68 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -123,7 +123,7 @@ func NewRelayerProxy( rp.sessionQuerier = sessiontypes.NewQueryClient(clientCtx) rp.applicationQuerier = apptypes.NewQueryClient(clientCtx) rp.keyring = rp.clientCtx.Keyring - rp.ringCache = make(map[string][]ringtypes.Point) + rp.ringCache = make(map[string][]ringtypes.Point) // the key is the appAddress rp.ringCacheMutex = &sync.RWMutex{} for _, opt := range opts { diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index 4238c1ce3..3846ac3af 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -20,7 +20,6 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { return err } - // TODO_DISCUSS: is there a reason not to assign rp.supplierAddress here? supplierAddress, err := supplierKey.GetAddress() if err != nil { return err diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index ef89e874c..08ea0f962 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -161,10 +161,11 @@ func (rs *relayerSessionsManager) mapBlockToSessionsToClaim( // Iterate over the sessionsTrees map to get the ones that end at a block height // lower than the current block height. for endBlockHeight, sessionsTreesEndingAtBlockHeight := range rs.sessionsTrees { - // TODO: We need this to be == instead of <= because we don't want to keep sending + // TODO_BLOCKER(@red-0ne): We need this to be == instead of <= because we don't want to keep sending // the same session while waiting the next step. This does not address the case // where the block client misses the target block which should be handled by the - // retry mechanism. + // retry mechanism. See the discussion in the following GitHub thread for next + // steps: https://github.com/pokt-network/poktroll/pull/177/files?show-viewed-files=true&file-filters%5B%5D=#r1391957041 if endBlockHeight == block.Height() { // Iterate over the sessionsTrees that end at this block height (or // less) and add them to the list of sessionTrees to be published. diff --git a/proto/pocket/service/relay.proto b/proto/pocket/service/relay.proto index dc269fad0..c9f0d5d77 100644 --- a/proto/pocket/service/relay.proto +++ b/proto/pocket/service/relay.proto @@ -34,6 +34,8 @@ message RelayRequest { } } +// TODO_TECHDEBT(#189, @h5law): See discussion related to #189 on how/why JSONRPC should be refactored altogether. + // JSONRPCRequestPayload contains the payload for a JSON-RPC request. // See https://www.jsonrpc.org/specification#request_object for more details. message JSONRPCRequestPayload { diff --git a/x/supplier/keeper/query_supplier.go b/x/supplier/keeper/query_supplier.go index 14d5afab5..7f381104c 100644 --- a/x/supplier/keeper/query_supplier.go +++ b/x/supplier/keeper/query_supplier.go @@ -2,6 +2,7 @@ package keeper import ( "context" + "fmt" "github.com/cosmos/cosmos-sdk/store/prefix" sdk "github.com/cosmos/cosmos-sdk/types" @@ -52,7 +53,9 @@ func (k Keeper) Supplier(goCtx context.Context, req *types.QueryGetSupplierReque req.Address, ) if !found { - return nil, status.Error(codes.NotFound, "not found") + // TODO_TECHDEBT(#181): conform to logging conventions once established + msg := fmt.Sprintf("supplier with address %q", req.GetAddress()) + return nil, status.Error(codes.NotFound, msg) } return &types.QueryGetSupplierResponse{Supplier: val}, nil diff --git a/x/supplier/keeper/query_supplier_test.go b/x/supplier/keeper/query_supplier_test.go index 6690e3f75..ad28a377b 100644 --- a/x/supplier/keeper/query_supplier_test.go +++ b/x/supplier/keeper/query_supplier_test.go @@ -47,7 +47,7 @@ func TestSupplierQuerySingle(t *testing.T) { request: &types.QueryGetSupplierRequest{ Address: strconv.Itoa(100000), }, - err: status.Error(codes.NotFound, "not found"), + err: status.Error(codes.NotFound, "supplier with address \"100000\""), }, { desc: "InvalidRequest",