From cec3ef41a00c952503a63e272e3bf41d433a3132 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Sat, 28 Oct 2023 01:58:52 +0200 Subject: [PATCH 01/32] feat: implement app client --- pkg/appclient/appclient.go | 159 +++++++++++++++++++++++++++++ pkg/appclient/endpoint_selector.go | 38 +++++++ pkg/appclient/errors.go | 9 ++ pkg/appclient/jsonrpc.go | 116 +++++++++++++++++++++ pkg/appclient/relay_verifier.go | 47 +++++++++ 5 files changed, 369 insertions(+) create mode 100644 pkg/appclient/appclient.go create mode 100644 pkg/appclient/endpoint_selector.go create mode 100644 pkg/appclient/errors.go create mode 100644 pkg/appclient/jsonrpc.go create mode 100644 pkg/appclient/relay_verifier.go diff --git a/pkg/appclient/appclient.go b/pkg/appclient/appclient.go new file mode 100644 index 000000000..611cc5466 --- /dev/null +++ b/pkg/appclient/appclient.go @@ -0,0 +1,159 @@ +package appclient + +import ( + "context" + "log" + "net/http" + "net/url" + "strings" + + sdkclient "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + + blocktypes "pocket/pkg/client" + "pocket/x/service/types" + sessiontypes "pocket/x/session/types" +) + +type appClient struct { + // keyName is the name of the key in the keyring that will be used to sign relay requests. + keyName string + keyring keyring.Keyring + + // clientCtx is the client context for the application. + // It is used to query for the application's account to unmarshal the supplier's account + // and get the public key to verify the relay response signature. + clientCtx sdkclient.Context + + // appAddress is the address of the application that this app client is for. + appAddress string + + // sessionQuerier is the querier for the session module. + // It used to get the current session for the application given a requested service. + sessionQuerier sessiontypes.QueryClient + + // accountQuerier is the querier for the account module. + // It is used to get the the supplier's public key to verify the relay response signature. + accountQuerier accounttypes.QueryClient + + // blockClient is the client for the block module. + // It is used to get the current block height to query for the current session. + blockClient blocktypes.BlockClient + + // server is the HTTP server that will be used capture application requests + // so that they can be signed and relayed to the supplier. + server *http.Server +} + +func NewAppClient( + clientCtx sdkclient.Context, + keyName string, + keyring keyring.Keyring, + applicationEndpoint *url.URL, + blockClient blocktypes.BlockClient, +) *appClient { + sessionQuerier := sessiontypes.NewQueryClient(clientCtx) + accountQuerier := accounttypes.NewQueryClient(clientCtx) + + return &appClient{ + clientCtx: clientCtx, + keyName: keyName, + keyring: keyring, + sessionQuerier: sessionQuerier, + accountQuerier: accountQuerier, + blockClient: blockClient, + server: &http.Server{Addr: applicationEndpoint.Host}, + } +} + +// Start starts the application server and blocks until the context is done +// or the server returns an error. +func (app *appClient) Start(ctx context.Context) error { + // Get and populate the application address from the keyring. + keyRecord, err := app.keyring.Key(app.keyName) + if err != nil { + return err + } + + accAddress, err := keyRecord.GetAddress() + if err != nil { + return err + } + + app.appAddress = accAddress.String() + + // Shutdown the HTTP server when the context is done. + go func() { + <-ctx.Done() + app.server.Shutdown(ctx) + }() + + // Start the HTTP server. + return app.server.ListenAndServe() +} + +// Stop stops the application server and returns any error that occurred. +func (app *appClient) Stop(ctx context.Context) error { + return app.server.Shutdown(ctx) +} + +// ServeHTTP is the HTTP handler for the application server. +// It captures the application request, signs it, and sends it to the supplier. +// After receiving the response from the supplier, it verifies the response signature +// before returning the response to the application. +// The serviceId is extracted from the request path. +func (app *appClient) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + ctx := request.Context() + + // Extract the serviceId from the request path. + path := request.URL.Path + serviceId := strings.Split(path, "/")[1] + + // Currently only JSON RPC requests are supported. + relayReponse, err := app.handleJSONRPCRelays(ctx, serviceId, request) + if err != nil { + // Reply with an error response if there was an error handling the relay. + app.replyWithError(writer, err) + log.Printf("ERROR: failed handling relay: %s", err) + return + } + + // Reply with the RelayResponse payload. + if _, err := writer.Write(relayReponse); err != nil { + app.replyWithError(writer, err) + return + } + log.Print("INFO: request serviced successfully") +} + +// replyWithError replies to the application with an error response. +// TODO_TECHDEBT: This method should be aware of the nature of the error to use the appropriate JSONRPC +// Code, Message and Data. Possibly by augmenting the passed in error with the adequate information. +func (app *appClient) replyWithError(writer http.ResponseWriter, err error) { + relayResponse := &types.RelayResponse{ + Payload: &types.RelayResponse_JsonRpcPayload{ + JsonRpcPayload: &types.JSONRPCResponsePayload{ + Id: make([]byte, 0), + Jsonrpc: "2.0", + Error: &types.JSONRPCResponseError{ + // Using conventional error code indicating internal server error. + Code: -32000, + Message: err.Error(), + Data: nil, + }, + }, + }, + } + + relayResponseBz, err := relayResponse.Marshal() + if err != nil { + log.Printf("ERROR: failed marshaling relay response: %s", err) + return + } + + if _, err = writer.Write(relayResponseBz); err != nil { + log.Printf("ERROR: failed writing relay response: %s", err) + return + } +} diff --git a/pkg/appclient/endpoint_selector.go b/pkg/appclient/endpoint_selector.go new file mode 100644 index 000000000..8455eceac --- /dev/null +++ b/pkg/appclient/endpoint_selector.go @@ -0,0 +1,38 @@ +package appclient + +import ( + "context" + "net/url" + + sessiontypes "pocket/x/session/types" + sharedtypes "pocket/x/shared/types" +) + +// getRelayerUrl gets the URL of the relayer for the given service. +// It gets the suppliers list from the current session and returns +// the first relayer URL that matches the JSON RPC RpcType. +func (app *appClient) getRelayerUrl( + ctx context.Context, + serviceId string, + session *sessiontypes.Session, +) (supplierUrl *url.URL, supplierAddress string, err error) { + for _, supplier := range session.Suppliers { + for _, service := range supplier.Services { + // Skip services that don't match the requested serviceId. + if service.ServiceId.Id != serviceId { + continue + } + + for _, endpoint := range service.Endpoints { + // Return the first endpoint url that matches the JSON RPC RpcType. + if endpoint.RpcType == sharedtypes.RPCType_JSON_RPC { + supplierUrl, err := url.Parse(endpoint.Url) + return supplierUrl, supplier.Address, err + } + } + } + } + + // Return an error if no relayer endpoints were found. + return nil, "", ErrNoRelayEndpoints +} diff --git a/pkg/appclient/errors.go b/pkg/appclient/errors.go new file mode 100644 index 000000000..e6721b074 --- /dev/null +++ b/pkg/appclient/errors.go @@ -0,0 +1,9 @@ +package appclient + +import sdkerrors "cosmossdk.io/errors" + +var ( + codespace = "appclient" + ErrInvalidRelayResponseSignature = sdkerrors.Register(codespace, 1, "invalid relay response signature") + ErrNoRelayEndpoints = sdkerrors.Register(codespace, 2, "no relay endpoints found") +) diff --git a/pkg/appclient/jsonrpc.go b/pkg/appclient/jsonrpc.go new file mode 100644 index 000000000..b27dc8d5e --- /dev/null +++ b/pkg/appclient/jsonrpc.go @@ -0,0 +1,116 @@ +package appclient + +import ( + "bytes" + "context" + "io" + "net/http" + + "github.com/cometbft/cometbft/crypto" + + "pocket/x/service/types" + sessiontypes "pocket/x/session/types" + sharedtypes "pocket/x/shared/types" +) + +// handleJSONRPCRelays handles JSON RPC relay requests. +func (app *appClient) handleJSONRPCRelays( + ctx context.Context, + serviceId string, + request *http.Request, +) (responseBz []byte, err error) { + // Read the request body bytes. + payloadBz, err := io.ReadAll(request.Body) + if err != nil { + return nil, err + } + + // Hash and sign the request payload. + hash := crypto.Sha256(payloadBz) + signature, _, err := app.keyring.Sign(app.keyName, hash) + if err != nil { + return nil, err + } + + // Create the relay request payload. + relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} + relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) + + // Get the current block height to query for the current session. + currentBlock := app.blockClient.LatestBlock(ctx) + + // Query for the current session. + sessionQueryReq := sessiontypes.QueryGetSessionRequest{ + ApplicationAddress: app.appAddress, + ServiceId: &sharedtypes.ServiceId{Id: serviceId}, + BlockHeight: currentBlock.Height(), + } + sessionQueryRes, err := app.sessionQuerier.GetSession(ctx, &sessionQueryReq) + if err != nil { + return nil, err + } + + session := sessionQueryRes.Session + + // Get a supplier URL and address for the given service and session. + supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, session) + if err != nil { + return nil, err + } + + // Create the relay request. + relayRequest := &types.RelayRequest{ + Meta: &types.RelayRequestMetadata{ + SessionHeader: session.Header, + Signature: signature, + }, + Payload: relayRequestPayload, + } + + // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. + relayRequestBz, err := relayRequest.Marshal() + if err != nil { + return nil, err + } + relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) + + // Create the HTTP request to send the request to the relayer. + relayHTTPRequest := &http.Request{ + Method: request.Method, + Header: request.Header, + URL: supplierUrl, + Body: relayRequestReader, + } + + // Perform the HTTP request to the relayer. + relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) + if err != nil { + return nil, err + } + + // Read the response body bytes. + relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) + if err != nil { + return nil, err + } + + // Unmarshal the response bytes into a RelayResponse. + relayResponse := &types.RelayResponse{} + if err := relayResponse.Unmarshal(relayResponseBz); err != nil { + return nil, err + } + + // Verify the response signature. We use the supplier address that we got from + // the getRelayerUrl function since this is the address we are expecting to sign the response. + // TODO_TECHDEBT: if the RelayResponse is an internal error response, we should not verify the signature + // as in some relayer early failures, it may not be signed by the supplier. + if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { + return nil, err + } + + // Marshal the response payload to bytes to be sent back to the application. + var responsePayloadBz []byte + _, err = relayResponse.Payload.MarshalTo(responsePayloadBz) + + return responsePayloadBz, err +} diff --git a/pkg/appclient/relay_verifier.go b/pkg/appclient/relay_verifier.go new file mode 100644 index 000000000..67b7a532d --- /dev/null +++ b/pkg/appclient/relay_verifier.go @@ -0,0 +1,47 @@ +package appclient + +import ( + "context" + + "github.com/cometbft/cometbft/crypto" + accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + + "pocket/x/service/types" +) + +// verifyResponse verifies the relay response signature. +func (app *appClient) verifyResponse( + ctx context.Context, + supplierAddress string, + relayResponse *types.RelayResponse, +) error { + // Query for the supplier account to get the application's public key to verify the relay request signature. + accQueryReq := &accounttypes.QueryAccountRequest{Address: supplierAddress} + accQueryRes, err := app.accountQuerier.Account(ctx, accQueryReq) + if err != nil { + return err + } + + // Marshal the relay response payload to bytes and get the hash. + var payloadBz []byte + _, err = relayResponse.Payload.MarshalTo(payloadBz) + if err != nil { + return err + } + hash := crypto.Sha256(payloadBz) + + // accQueryRes.Account.Value is a protobuf Any type that should be unmarshaled into an AccountI interface. + // TODO_TECHDEBT: Make sure our `AccountI`/`any` unmarshalling is correct. + // See https://github.com/pokt-network/poktroll/pull/101/files/edbd628e9146e232ef58c71cfa8f4be2135cdb50..fbba10626df79f6bf6e2218513dfdeb40a629790#r1372464439 + var account accounttypes.AccountI + if err := app.clientCtx.Codec.UnmarshalJSON(accQueryRes.Account.Value, account); err != nil { + return err + } + + // Verify the relay response signature. + if !account.GetPubKey().VerifySignature(hash, relayResponse.Meta.SupplierSignature) { + return ErrInvalidRelayResponseSignature + } + + return nil +} From c4fe2da2ba665d8059278234c772a1f76fa67d4f Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 7 Nov 2023 02:49:04 +0100 Subject: [PATCH 02/32] chore: address review comments --- pkg/appclient/relay_verifier.go | 47 ------------ .../endpoint_selector.go | 23 +++--- pkg/{appclient => appserver}/errors.go | 2 +- pkg/{appclient => appserver}/jsonrpc.go | 58 +++++++-------- pkg/appserver/relay_verifier.go | 69 ++++++++++++++++++ .../appclient.go => appserver/server.go} | 72 ++++++++++++------- pkg/appserver/session.go | 47 ++++++++++++ 7 files changed, 204 insertions(+), 114 deletions(-) delete mode 100644 pkg/appclient/relay_verifier.go rename pkg/{appclient => appserver}/endpoint_selector.go (54%) rename pkg/{appclient => appserver}/errors.go (94%) rename pkg/{appclient => appserver}/jsonrpc.go (70%) create mode 100644 pkg/appserver/relay_verifier.go rename pkg/{appclient/appclient.go => appserver/server.go} (62%) create mode 100644 pkg/appserver/session.go diff --git a/pkg/appclient/relay_verifier.go b/pkg/appclient/relay_verifier.go deleted file mode 100644 index 67b7a532d..000000000 --- a/pkg/appclient/relay_verifier.go +++ /dev/null @@ -1,47 +0,0 @@ -package appclient - -import ( - "context" - - "github.com/cometbft/cometbft/crypto" - accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" - - "pocket/x/service/types" -) - -// verifyResponse verifies the relay response signature. -func (app *appClient) verifyResponse( - ctx context.Context, - supplierAddress string, - relayResponse *types.RelayResponse, -) error { - // Query for the supplier account to get the application's public key to verify the relay request signature. - accQueryReq := &accounttypes.QueryAccountRequest{Address: supplierAddress} - accQueryRes, err := app.accountQuerier.Account(ctx, accQueryReq) - if err != nil { - return err - } - - // Marshal the relay response payload to bytes and get the hash. - var payloadBz []byte - _, err = relayResponse.Payload.MarshalTo(payloadBz) - if err != nil { - return err - } - hash := crypto.Sha256(payloadBz) - - // accQueryRes.Account.Value is a protobuf Any type that should be unmarshaled into an AccountI interface. - // TODO_TECHDEBT: Make sure our `AccountI`/`any` unmarshalling is correct. - // See https://github.com/pokt-network/poktroll/pull/101/files/edbd628e9146e232ef58c71cfa8f4be2135cdb50..fbba10626df79f6bf6e2218513dfdeb40a629790#r1372464439 - var account accounttypes.AccountI - if err := app.clientCtx.Codec.UnmarshalJSON(accQueryRes.Account.Value, account); err != nil { - return err - } - - // Verify the relay response signature. - if !account.GetPubKey().VerifySignature(hash, relayResponse.Meta.SupplierSignature) { - return ErrInvalidRelayResponseSignature - } - - return nil -} diff --git a/pkg/appclient/endpoint_selector.go b/pkg/appserver/endpoint_selector.go similarity index 54% rename from pkg/appclient/endpoint_selector.go rename to pkg/appserver/endpoint_selector.go index 8455eceac..aff1e916e 100644 --- a/pkg/appclient/endpoint_selector.go +++ b/pkg/appserver/endpoint_selector.go @@ -1,19 +1,22 @@ -package appclient +package appserver import ( "context" + "log" "net/url" - sessiontypes "pocket/x/session/types" - sharedtypes "pocket/x/shared/types" + sessiontypes "github.com/pokt-network/poktroll/x/session/types" + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) +// TODO_IMPROVE: This implements a naive greedy approach that defaults to the +// first available URL but future optimizations can be introduced. +// TODO(@h5law): Look into different endpoint selection depending on their suitability. // getRelayerUrl gets the URL of the relayer for the given service. -// It gets the suppliers list from the current session and returns -// the first relayer URL that matches the JSON RPC RpcType. -func (app *appClient) getRelayerUrl( +func (app *appServer) getRelayerUrl( ctx context.Context, serviceId string, + rpcType sharedtypes.RPCType, session *sessiontypes.Session, ) (supplierUrl *url.URL, supplierAddress string, err error) { for _, supplier := range session.Suppliers { @@ -25,9 +28,13 @@ func (app *appClient) getRelayerUrl( for _, endpoint := range service.Endpoints { // Return the first endpoint url that matches the JSON RPC RpcType. - if endpoint.RpcType == sharedtypes.RPCType_JSON_RPC { + if endpoint.RpcType == rpcType { supplierUrl, err := url.Parse(endpoint.Url) - return supplierUrl, supplier.Address, err + if err != nil { + log.Printf("error parsing url: %s", err) + continue + } + return supplierUrl, supplier.Address, nil } } } diff --git a/pkg/appclient/errors.go b/pkg/appserver/errors.go similarity index 94% rename from pkg/appclient/errors.go rename to pkg/appserver/errors.go index e6721b074..722c6aaf8 100644 --- a/pkg/appclient/errors.go +++ b/pkg/appserver/errors.go @@ -1,4 +1,4 @@ -package appclient +package appserver import sdkerrors "cosmossdk.io/errors" diff --git a/pkg/appclient/jsonrpc.go b/pkg/appserver/jsonrpc.go similarity index 70% rename from pkg/appclient/jsonrpc.go rename to pkg/appserver/jsonrpc.go index b27dc8d5e..0f4370300 100644 --- a/pkg/appclient/jsonrpc.go +++ b/pkg/appserver/jsonrpc.go @@ -1,4 +1,4 @@ -package appclient +package appserver import ( "bytes" @@ -8,54 +8,43 @@ import ( "github.com/cometbft/cometbft/crypto" - "pocket/x/service/types" - sessiontypes "pocket/x/session/types" - sharedtypes "pocket/x/shared/types" + "github.com/pokt-network/poktroll/x/service/types" + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) -// handleJSONRPCRelays handles JSON RPC relay requests. -func (app *appClient) handleJSONRPCRelays( +// handleJSONRPCRelay handles JSON RPC relay requests. +func (app *appServer) handleJSONRPCRelay( ctx context.Context, serviceId string, request *http.Request, -) (responseBz []byte, err error) { + writer http.ResponseWriter, +) error { // Read the request body bytes. payloadBz, err := io.ReadAll(request.Body) if err != nil { - return nil, err + return err } // Hash and sign the request payload. hash := crypto.Sha256(payloadBz) signature, _, err := app.keyring.Sign(app.keyName, hash) if err != nil { - return nil, err + return err } // Create the relay request payload. relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) - // Get the current block height to query for the current session. - currentBlock := app.blockClient.LatestBlock(ctx) - - // Query for the current session. - sessionQueryReq := sessiontypes.QueryGetSessionRequest{ - ApplicationAddress: app.appAddress, - ServiceId: &sharedtypes.ServiceId{Id: serviceId}, - BlockHeight: currentBlock.Height(), - } - sessionQueryRes, err := app.sessionQuerier.GetSession(ctx, &sessionQueryReq) + session, err := app.getCurrentSession(ctx, serviceId) if err != nil { - return nil, err + return err } - session := sessionQueryRes.Session - // Get a supplier URL and address for the given service and session. - supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, session) + supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, sharedtypes.RPCType_JSON_RPC, session) if err != nil { - return nil, err + return err } // Create the relay request. @@ -70,7 +59,7 @@ func (app *appClient) handleJSONRPCRelays( // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := relayRequest.Marshal() if err != nil { - return nil, err + return err } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) @@ -85,19 +74,19 @@ func (app *appClient) handleJSONRPCRelays( // Perform the HTTP request to the relayer. relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { - return nil, err + return err } // Read the response body bytes. relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { - return nil, err + return err } // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} if err := relayResponse.Unmarshal(relayResponseBz); err != nil { - return nil, err + return err } // Verify the response signature. We use the supplier address that we got from @@ -105,12 +94,19 @@ func (app *appClient) handleJSONRPCRelays( // TODO_TECHDEBT: if the RelayResponse is an internal error response, we should not verify the signature // as in some relayer early failures, it may not be signed by the supplier. if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { - return nil, err + return err } // Marshal the response payload to bytes to be sent back to the application. var responsePayloadBz []byte - _, err = relayResponse.Payload.MarshalTo(responsePayloadBz) + if _, err = relayResponse.Payload.MarshalTo(responsePayloadBz); err != nil { + return err + } + + // Reply with the RelayResponse payload. + if _, err := writer.Write(relayRequestBz); err != nil { + return err + } - return responsePayloadBz, err + return nil } diff --git a/pkg/appserver/relay_verifier.go b/pkg/appserver/relay_verifier.go new file mode 100644 index 000000000..1c3b2ada5 --- /dev/null +++ b/pkg/appserver/relay_verifier.go @@ -0,0 +1,69 @@ +package appserver + +import ( + "context" + + "github.com/cometbft/cometbft/crypto" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + + "github.com/pokt-network/poktroll/x/service/types" +) + +// verifyResponse verifies the relay response signature. +func (app *appServer) verifyResponse( + ctx context.Context, + supplierAddress string, + relayResponse *types.RelayResponse, +) error { + pubKey, err := app.getSupplierPubKeyFromAddress(ctx, supplierAddress) + if err != nil { + return err + } + + // Marshal the relay response payload to bytes and get the hash. + var payloadBz []byte + if _, err = relayResponse.Payload.MarshalTo(payloadBz); err != nil { + return err + } + hash := crypto.Sha256(payloadBz) + + // Verify the relay response signature. + if !pubKey.VerifySignature(hash, relayResponse.Meta.SupplierSignature) { + return ErrInvalidRelayResponseSignature + } + + return nil +} + +// getSupplierPubKeyFromAddress gets the supplier's public key from the cache or queries +// if it is not found. +// The public key is then cached before being returned. +func (app *appServer) getSupplierPubKeyFromAddress( + ctx context.Context, + supplierAddress string, +) (cryptotypes.PubKey, error) { + pubKey, ok := app.supplierAccountCache[supplierAddress] + if ok { + return pubKey, nil + } + + // Query for the supplier account to get the application's public key to verify the relay request signature. + accQueryReq := &accounttypes.QueryAccountRequest{Address: supplierAddress} + accQueryRes, err := app.accountQuerier.Account(ctx, accQueryReq) + if err != nil { + return nil, err + } + + // Unmarshal the query response into a BaseAccount. + account := new(accounttypes.BaseAccount) + if err := account.Unmarshal(accQueryRes.Account.Value); err != nil { + return nil, err + } + + fetchedPubKey := account.GetPubKey() + // Cache the retrieved public key. + app.supplierAccountCache[supplierAddress] = fetchedPubKey + + return fetchedPubKey, nil +} diff --git a/pkg/appclient/appclient.go b/pkg/appserver/server.go similarity index 62% rename from pkg/appclient/appclient.go rename to pkg/appserver/server.go index 611cc5466..88a9e64b7 100644 --- a/pkg/appclient/appclient.go +++ b/pkg/appserver/server.go @@ -1,4 +1,4 @@ -package appclient +package appserver import ( "context" @@ -6,17 +6,23 @@ import ( "net/http" "net/url" "strings" + "sync" sdkclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/crypto/keyring" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" - blocktypes "pocket/pkg/client" - "pocket/x/service/types" - sessiontypes "pocket/x/session/types" + blocktypes "github.com/pokt-network/poktroll/pkg/client" + "github.com/pokt-network/poktroll/x/service/types" + sessiontypes "github.com/pokt-network/poktroll/x/session/types" ) -type appClient struct { +// appServer is the server that listens for application requests and relays them to the supplier. +// it is responsible for maintaining the current session for the application, signing the requests, +// and verifying the response signatures. +type appServer struct { + // TODO(@h5law): Replace with ring signature. // keyName is the name of the key in the keyring that will be used to sign relay requests. keyName string keyring keyring.Keyring @@ -26,13 +32,20 @@ type appClient struct { // and get the public key to verify the relay response signature. clientCtx sdkclient.Context - // appAddress is the address of the application that this app client is for. + // appAddress is the address of the application that this app server is running for. appAddress string // sessionQuerier is the querier for the session module. // It used to get the current session for the application given a requested service. sessionQuerier sessiontypes.QueryClient + // sessionMu is a mutex to protect currentSession map reads and and updates. + sessionMu sync.RWMutex + + // currentSessions is the current session for the application given a block height. + // It is updated by the goListenForNewSessions goroutine. + currentSessions map[string]*sessiontypes.Session + // accountQuerier is the querier for the account module. // It is used to get the the supplier's public key to verify the relay response signature. accountQuerier accounttypes.QueryClient @@ -44,32 +57,37 @@ type appClient struct { // server is the HTTP server that will be used capture application requests // so that they can be signed and relayed to the supplier. server *http.Server + + // accountCache is a cache of the supplier accounts that has been queried + // TODO_TECHDEBT: Add a size limit to the cache. + supplierAccountCache map[string]cryptotypes.PubKey } -func NewAppClient( +func NewAppServer( clientCtx sdkclient.Context, keyName string, keyring keyring.Keyring, applicationEndpoint *url.URL, blockClient blocktypes.BlockClient, -) *appClient { +) *appServer { sessionQuerier := sessiontypes.NewQueryClient(clientCtx) accountQuerier := accounttypes.NewQueryClient(clientCtx) - return &appClient{ - clientCtx: clientCtx, - keyName: keyName, - keyring: keyring, - sessionQuerier: sessionQuerier, - accountQuerier: accountQuerier, - blockClient: blockClient, - server: &http.Server{Addr: applicationEndpoint.Host}, + return &appServer{ + clientCtx: clientCtx, + keyName: keyName, + keyring: keyring, + sessionQuerier: sessionQuerier, + accountQuerier: accountQuerier, + blockClient: blockClient, + server: &http.Server{Addr: applicationEndpoint.Host}, + supplierAccountCache: make(map[string]cryptotypes.PubKey), } } // Start starts the application server and blocks until the context is done // or the server returns an error. -func (app *appClient) Start(ctx context.Context) error { +func (app *appServer) Start(ctx context.Context) error { // Get and populate the application address from the keyring. keyRecord, err := app.keyring.Key(app.keyName) if err != nil { @@ -94,7 +112,7 @@ func (app *appClient) Start(ctx context.Context) error { } // Stop stops the application server and returns any error that occurred. -func (app *appClient) Stop(ctx context.Context) error { +func (app *appServer) Stop(ctx context.Context) error { return app.server.Shutdown(ctx) } @@ -103,15 +121,20 @@ func (app *appClient) Stop(ctx context.Context) error { // After receiving the response from the supplier, it verifies the response signature // before returning the response to the application. // The serviceId is extracted from the request path. -func (app *appClient) ServeHTTP(writer http.ResponseWriter, request *http.Request) { +// The request's path should be of the form "/{serviceId}/[other/path/segments]", +// where the serviceId is the id of the service that the application is requesting +// and the other (possible) path segments are the JSON RPC request path. +func (app *appServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { ctx := request.Context() // Extract the serviceId from the request path. path := request.URL.Path serviceId := strings.Split(path, "/")[1] - // Currently only JSON RPC requests are supported. - relayReponse, err := app.handleJSONRPCRelays(ctx, serviceId, request) + // TODO_TECHDEBT: Currently, there is no information about the RPC type requested. It should + // be extracted from the request and used to determine the RPC type to handle. handle*Relay() + // calls should be wrapped into a switch statement to handle different types of relays. + err := app.handleJSONRPCRelay(ctx, serviceId, request, writer) if err != nil { // Reply with an error response if there was an error handling the relay. app.replyWithError(writer, err) @@ -119,18 +142,13 @@ func (app *appClient) ServeHTTP(writer http.ResponseWriter, request *http.Reques return } - // Reply with the RelayResponse payload. - if _, err := writer.Write(relayReponse); err != nil { - app.replyWithError(writer, err) - return - } log.Print("INFO: request serviced successfully") } // replyWithError replies to the application with an error response. // TODO_TECHDEBT: This method should be aware of the nature of the error to use the appropriate JSONRPC // Code, Message and Data. Possibly by augmenting the passed in error with the adequate information. -func (app *appClient) replyWithError(writer http.ResponseWriter, err error) { +func (app *appServer) replyWithError(writer http.ResponseWriter, err error) { relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ diff --git a/pkg/appserver/session.go b/pkg/appserver/session.go new file mode 100644 index 000000000..c3b64782f --- /dev/null +++ b/pkg/appserver/session.go @@ -0,0 +1,47 @@ +package appserver + +import ( + "context" + + sessiontypes "github.com/pokt-network/poktroll/x/session/types" + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" +) + +// getCurrentSession gets the current session for the given service +// It returns the current session if it exists and is still valid, otherwise it +// queries for the latest session, caches and returns it. +func (app *appServer) getCurrentSession( + ctx context.Context, + serviceId string, +) (*sessiontypes.Session, error) { + app.sessionMu.RLock() + defer app.sessionMu.RUnlock() + + latestBlock := app.blockClient.LatestBlock(ctx) + if currentSession, ok := app.currentSessions[serviceId]; ok { + sessionEndBlockHeight := currentSession.Header.SessionStartBlockHeight + currentSession.NumBlocksPerSession + + // Return the current session if it is still valid. + if latestBlock.Height() < sessionEndBlockHeight { + return currentSession, nil + } + } + + // Query for the current session. + sessionQueryReq := sessiontypes.QueryGetSessionRequest{ + ApplicationAddress: app.appAddress, + ServiceId: &sharedtypes.ServiceId{Id: serviceId}, + BlockHeight: latestBlock.Height(), + } + sessionQueryRes, err := app.sessionQuerier.GetSession(ctx, &sessionQueryReq) + if err != nil { + return nil, err + } + + session := sessionQueryRes.Session + + // Cache the current session. + app.currentSessions[serviceId] = session + + return session, nil +} From a6edf6175b50c12353ad9c2047780e5127486f28 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 7 Nov 2023 02:54:02 +0100 Subject: [PATCH 03/32] Merge remote-tracking branch 'origin/main' into feat/app-client --- .github/label-actions.yml | 35 +++++ .github/workflows/go.yml | 54 +++++++- .github/workflows/label-actions.yml | 21 +++ .gitignore | 6 +- Dockerfile.dev | 23 ++++ Makefile | 37 +++-- Tiltfile | 6 +- e2e/tests/node.go | 2 +- internal/testclient/testeventsquery/client.go | 43 ++++++ pkg/client/block/client.go | 2 +- pkg/client/block/client_test.go | 129 +++++++++--------- .../client/cli/tx_delegate_to_gateway.go | 2 +- .../client/cli/tx_stake_application.go | 2 +- .../client/cli/tx_undelegate_from_gateway.go | 2 +- .../client/cli/tx_unstake_application.go | 2 +- x/gateway/client/cli/tx_stake_gateway.go | 2 +- x/gateway/client/cli/tx_unstake_gateway.go | 2 +- x/supplier/client/cli/tx_stake_supplier.go | 2 +- x/supplier/client/cli/tx_unstake_supplier.go | 2 +- 19 files changed, 276 insertions(+), 98 deletions(-) create mode 100644 .github/label-actions.yml create mode 100644 .github/workflows/label-actions.yml create mode 100644 Dockerfile.dev diff --git a/.github/label-actions.yml b/.github/label-actions.yml new file mode 100644 index 000000000..b4dc1814d --- /dev/null +++ b/.github/label-actions.yml @@ -0,0 +1,35 @@ +# When `devnet-e2e-test` is added, also assign `devnet` to the PR. +devnet-e2e-test: + prs: + comment: The CI will now also run the e2e tests on devnet, which increases the time it takes to complete all CI checks. + label: + - devnet + +# When `devnet-e2e-test` is removed, also delete `devnet` from the PR. +-devnet-e2e-test: + prs: + unlabel: + - devnet + +# When `devnet` is added, also assign `push-image` to the PR. +devnet: + prs: + label: + - push-image + +# When `devnet` is removed, also delete `devnet-e2e-test` from the PR. +-devnet: + prs: + unlabel: + - devnet-e2e-test + +# Let the developer know that they need to push another commit after attaching the label to PR. +push-image: + prs: + comment: The image is going to be pushed after the next commit. If you want to run an e2e test, it is necessary to push another commit. You can use `make trigger_ci` to push an empty commit. + +# When `push-image` is removed, also delete `devnet` from the PR. +-push-image: + prs: + unlabel: + - devnet diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1b60ddc4c..6f76334c1 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,11 +8,16 @@ on: branches: ["main"] pull_request: +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest steps: - name: install ignite + # If this step fails due to ignite.com failing, see #116 for a temporary workaround run: | curl https://get.ignite.com/cli! | bash ignite version @@ -39,7 +44,54 @@ jobs: run: make go_lint - name: Build - run: ignite chain build --debug --skip-proto + run: ignite chain build -v --debug --skip-proto - name: Test run: make go_test + + - name: Set up Docker Buildx + if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) + uses: docker/setup-buildx-action@v3 + + - name: Docker Metadata action + if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_PR_HEAD_SHA: "true" + with: + images: | + ghcr.io/pokt-network/pocketd + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha + type=sha,format=long + + - name: Login to GitHub Container Registry + if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Copy binary to inside of the Docker context + if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) + run: | + mkdir -p ./bin # Make sure the bin directory exists + cp $(go env GOPATH)/bin/poktrolld ./bin # Copy the binary to the repo's bin directory + + - name: Build and push Docker image + if: (github.ref == 'refs/heads/main') || (contains(github.event.pull_request.labels.*.name, 'push-image')) + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # NB: Uncomment below if arm64 build is needed; arm64 builds are off by default because build times are significant. + platforms: linux/amd64 #,linux/arm64 + file: Dockerfile.dev + cache-from: type=gha + cache-to: type=gha,mode=max + context: . diff --git a/.github/workflows/label-actions.yml b/.github/workflows/label-actions.yml new file mode 100644 index 000000000..caf1a31cc --- /dev/null +++ b/.github/workflows/label-actions.yml @@ -0,0 +1,21 @@ +name: 'Label Actions' + +on: + issues: + types: [labeled, unlabeled] + pull_request_target: + types: [labeled, unlabeled] + discussion: + types: [labeled, unlabeled] + +permissions: + contents: read + issues: write + pull-requests: write + discussions: write + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/label-actions@v3 diff --git a/.gitignore b/.gitignore index aa7066b08..5dc239e58 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,6 @@ go.work # Don't commit binaries bin -!bin/.keep # Before we provision the localnet, `ignite` creates the accounts, genesis, etc. for us # As many of the files are dynamic, we only preserve the config files in git history. @@ -57,4 +56,7 @@ ts-client/ **/*_mock.go # Localnet config -localnet_config.yaml \ No newline at end of file +localnet_config.yaml + +# Relase artifacts produced by `ignite chain build --release` +release diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..2d10955e0 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,23 @@ +# This Dockerfile is used to build container image for development purposes. +# It intentionally contains no security features, ships with code and troubleshooting tools. + +FROM golang:1.20 as base + +RUN apt update && \ + apt-get install -y \ + ca-certificates \ + curl jq make + +# enable faster module downloading. +ENV GOPROXY https://proxy.golang.org + +COPY . /poktroll + +WORKDIR /poktroll + +RUN mv /poktroll/bin/poktrolld /usr/bin/poktrolld + +EXPOSE 8545 +EXPOSE 8546 + +ENTRYPOINT ["ignite"] diff --git a/Makefile b/Makefile index d6d3257dc..19c8fe59e 100644 --- a/Makefile +++ b/Makefile @@ -231,11 +231,11 @@ todo_this_commit: ## List all the TODOs needed to be done in this commit .PHONY: gateway_list gateway_list: ## List all the staked gateways - pocketd --home=$(POCKETD_HOME) q gateway list-gateway --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q gateway list-gateway --node $(POCKET_NODE) .PHONY: gateway_stake gateway_stake: ## Stake tokens for the gateway specified (must specify the gateway env var) - pocketd --home=$(POCKETD_HOME) tx gateway stake-gateway 1000upokt --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx gateway stake-gateway 1000upokt --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE) .PHONY: gateway1_stake gateway1_stake: ## Stake gateway1 @@ -251,7 +251,7 @@ gateway3_stake: ## Stake gateway3 .PHONY: gateway_unstake gateway_unstake: ## Unstake an gateway (must specify the GATEWAY env var) - pocketd --home=$(POCKETD_HOME) tx gateway unstake-gateway --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx gateway unstake-gateway --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE) .PHONY: gateway1_unstake gateway1_unstake: ## Unstake gateway1 @@ -271,11 +271,11 @@ gateway3_unstake: ## Unstake gateway3 .PHONY: app_list app_list: ## List all the staked applications - pocketd --home=$(POCKETD_HOME) q application list-application --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q application list-application --node $(POCKET_NODE) .PHONY: app_stake app_stake: ## Stake tokens for the application specified (must specify the APP and SERVICES env vars) - pocketd --home=$(POCKETD_HOME) tx application stake-application 1000upokt $(SERVICES) --keyring-backend test --from $(APP) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx application stake-application 1000upokt $(SERVICES) --keyring-backend test --from $(APP) --node $(POCKET_NODE) .PHONY: app1_stake app1_stake: ## Stake app1 @@ -291,7 +291,7 @@ app3_stake: ## Stake app3 .PHONY: app_unstake app_unstake: ## Unstake an application (must specify the APP env var) - pocketd --home=$(POCKETD_HOME) tx application unstake-application --keyring-backend test --from $(APP) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx application unstake-application --keyring-backend test --from $(APP) --node $(POCKET_NODE) .PHONY: app1_unstake app1_unstake: ## Unstake app1 @@ -307,7 +307,7 @@ app3_unstake: ## Unstake app3 .PHONY: app_delegate app_delegate: ## Delegate trust to a gateway (must specify the APP and GATEWAY_ADDR env vars). Requires the app to be staked - pocketd --home=$(POCKETD_HOME) tx application delegate-to-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx application delegate-to-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE) .PHONY: app1_delegate_gateway1 app1_delegate_gateway1: ## Delegate trust to gateway1 @@ -323,7 +323,7 @@ app3_delegate_gateway3: ## Delegate trust to gateway3 .PHONY: app_undelegate app_undelegate: ## Undelegate trust to a gateway (must specify the APP and GATEWAY_ADDR env vars). Requires the app to be staked - pocketd --home=$(POCKETD_HOME) tx application undelegate-from-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx application undelegate-from-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE) .PHONY: app1_undelegate_gateway1 app1_undelegate_gateway1: ## Undelegate trust to gateway1 @@ -343,13 +343,13 @@ app3_undelegate_gateway3: ## Undelegate trust to gateway3 .PHONY: supplier_list supplier_list: ## List all the staked supplier - pocketd --home=$(POCKETD_HOME) q supplier list-supplier --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q supplier list-supplier --node $(POCKET_NODE) # TODO(@Olshansk, @okdas): Add more services (in addition to anvil) for apps and suppliers to stake for. # TODO_TECHDEBT: svc1, svc2 and svc3 below are only in place to make GetSession testable .PHONY: supplier_stake supplier_stake: ## Stake tokens for the supplier specified (must specify the APP env var) - pocketd --home=$(POCKETD_HOME) tx supplier stake-supplier 1000upokt "$(SERVICES)" --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx supplier stake-supplier 1000upokt "$(SERVICES)" --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) .PHONY: supplier1_stake supplier1_stake: ## Stake supplier1 @@ -365,7 +365,7 @@ supplier3_stake: ## Stake supplier3 .PHONY: supplier_unstake supplier_unstake: ## Unstake an supplier (must specify the SUPPLIER env var) - pocketd --home=$(POCKETD_HOME) tx supplier unstake-supplier --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) tx supplier unstake-supplier --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) .PHONY: supplier1_unstake supplier1_unstake: ## Unstake supplier1 @@ -386,10 +386,10 @@ supplier3_unstake: ## Unstake supplier3 .PHONY: acc_balance_query acc_balance_query: ## Query the balance of the account specified (make acc_balance_query ACC=pokt...) @echo "~~~ Balances ~~~" - pocketd --home=$(POCKETD_HOME) q bank balances $(ACC) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q bank balances $(ACC) --node $(POCKET_NODE) @echo "~~~ Spendable Balances ~~~" @echo "Querying spendable balance for $(ACC)" - pocketd --home=$(POCKETD_HOME) q bank spendable-balances $(ACC) --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q bank spendable-balances $(ACC) --node $(POCKET_NODE) .PHONY: acc_balance_query_module_app acc_balance_query_module_app: ## Query the balance of the network level "application" module @@ -405,7 +405,7 @@ acc_balance_query_app1: ## Query the balance of app1 .PHONY: acc_balance_total_supply acc_balance_total_supply: ## Query the total supply of the network - pocketd --home=$(POCKETD_HOME) q bank total --node $(POCKET_NODE) + poktrolld --home=$(POCKETD_HOME) q bank total --node $(POCKET_NODE) ###################### ### Ignite Helpers ### @@ -415,6 +415,15 @@ acc_balance_total_supply: ## Query the total supply of the network ignite_acc_list: ## List all the accounts in LocalNet ignite account list --keyring-dir=$(POCKETD_HOME) --keyring-backend test --address-prefix $(POCKET_ADDR_PREFIX) +################## +### CI Helpers ### +################## + +.PHONY: trigger_ci +trigger_ci: ## Trigger the CI pipeline by submitting an empty commit; See https://github.com/pokt-network/pocket/issues/900 for details + git commit --allow-empty -m "Empty commit" + git push + ##################### ### Documentation ### ##################### diff --git a/Tiltfile b/Tiltfile index 1fd0dd779..17521b14b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -101,12 +101,12 @@ docker_build_with_restart( dockerfile_contents="""FROM golang:1.20.8 RUN apt-get -q update && apt-get install -qyy curl jq RUN go install github.com/go-delve/delve/cmd/dlv@latest -COPY bin/pocketd /usr/local/bin/pocketd +COPY bin/poktrolld /usr/local/bin/pocketd WORKDIR / """, - only=["./bin/pocketd"], + only=["./bin/poktrolld"], entrypoint=["/bin/sh", "/scripts/pocket.sh"], - live_update=[sync("bin/pocketd", "/usr/local/bin/pocketd")], + live_update=[sync("bin/poktrolld", "/usr/local/bin/pocketd")], ) # Run celestia and anvil nodes diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 4e34fa827..e46ad2889 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -67,7 +67,7 @@ func (p *pocketdBin) RunCommandOnHost(rpcUrl string, args ...string) (*commandRe func (p *pocketdBin) runCmd(args ...string) (*commandResult, error) { base := []string{"--home", defaultHome} args = append(base, args...) - cmd := exec.Command("pocketd", args...) + cmd := exec.Command("poktrolld", args...) r := &commandResult{} out, err := cmd.Output() if err != nil { diff --git a/internal/testclient/testeventsquery/client.go b/internal/testclient/testeventsquery/client.go index d55a765ab..0aa618fe9 100644 --- a/internal/testclient/testeventsquery/client.go +++ b/internal/testclient/testeventsquery/client.go @@ -1,11 +1,18 @@ package testeventsquery import ( + "context" "testing" + "time" + "github.com/golang/mock/gomock" + + "github.com/pokt-network/poktroll/internal/mocks/mockclient" "github.com/pokt-network/poktroll/internal/testclient" "github.com/pokt-network/poktroll/pkg/client" eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" + "github.com/pokt-network/poktroll/pkg/either" + "github.com/pokt-network/poktroll/pkg/observable/channel" ) // NewLocalnetClient returns a new events query client which is configured to @@ -15,3 +22,39 @@ func NewLocalnetClient(t *testing.T, opts ...client.EventsQueryClientOption) cli return eventsquery.NewEventsQueryClient(testclient.CometLocalWebsocketURL, opts...) } + +// NewAnyTimesEventsBytesEventsQueryClient returns a new events query client which +// is configured to return the expected event bytes when queried with the expected +// query, any number of times. The returned client also expects to be closed once. +func NewAnyTimesEventsBytesEventsQueryClient( + ctx context.Context, + t *testing.T, + expectedQuery string, + expectedEventBytes []byte, +) client.EventsQueryClient { + t.Helper() + + ctrl := gomock.NewController(t) + eventsQueryClient := mockclient.NewMockEventsQueryClient(ctrl) + eventsQueryClient.EXPECT().Close().Times(1) + eventsQueryClient.EXPECT(). + EventsBytes(gomock.AssignableToTypeOf(ctx), gomock.Eq(expectedQuery)). + DoAndReturn( + func(ctx context.Context, query string) (client.EventsBytesObservable, error) { + bytesObsvbl, bytesPublishCh := channel.NewReplayObservable[either.Bytes](ctx, 1) + + // Now that the observable is set up, publish the expected event bytes. + // Only need to send once because it's a ReplayObservable. + bytesPublishCh <- either.Success(expectedEventBytes) + + // Wait a tick for the observables to be set up. This isn't strictly + // necessary but is done to mitigate test flakiness. + time.Sleep(10 * time.Millisecond) + + return bytesObsvbl, nil + }, + ). + AnyTimes() + + return eventsQueryClient +} diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index 54569e60d..18526508d 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -155,7 +155,7 @@ func (bClient *blockClient) retryPublishBlocksFactory(ctx context.Context) func( } // NB: must cast back to generic observable type to use with Map. - // client.BlocksObservable is only used to workaround gomock's lack of + // client.BlocksObservable cannot be an alias due to gomock's lack of // support for generic types. eventsBz := observable.Observable[either.Either[[]byte]](eventsBzObsvbl) blockEventFromEventBz := newEventsBytesToBlockMapFn(errCh) diff --git a/pkg/client/block/client_test.go b/pkg/client/block/client_test.go index b983ff274..b2a5515b3 100644 --- a/pkg/client/block/client_test.go +++ b/pkg/client/block/client_test.go @@ -8,17 +8,20 @@ import ( "cosmossdk.io/depinject" comettypes "github.com/cometbft/cometbft/types" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" "github.com/pokt-network/poktroll/internal/testclient" "github.com/pokt-network/poktroll/internal/testclient/testeventsquery" "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/client/block" - eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" ) -const blockAssertionLoopTimeout = 500 * time.Millisecond +const ( + testTimeoutDuration = 100 * time.Millisecond + + // duplicates pkg/client/block/client.go's committedBlocksQuery for testing purposes + committedBlocksQuery = "tm.event='NewBlock'" +) func TestBlockClient(t *testing.T) { var ( @@ -38,19 +41,15 @@ func TestBlockClient(t *testing.T) { ctx = context.Background() ) - // Set up a mock connection and dialer which are expected to be used once. - connMock, dialerMock := testeventsquery.NewOneTimeMockConnAndDialer(t) - connMock.EXPECT().Send(gomock.Any()).Return(nil).Times(1) - // Mock the Receive method to return the expected block event. - connMock.EXPECT().Receive().DoAndReturn(func() ([]byte, error) { - blockEventJson, err := json.Marshal(expectedBlockEvent) - require.NoError(t, err) - return blockEventJson, nil - }).AnyTimes() - - // Set up events query client dependency. - dialerOpt := eventsquery.WithDialer(dialerMock) - eventsQueryClient := testeventsquery.NewLocalnetClient(t, dialerOpt) + expectedEventBz, err := json.Marshal(expectedBlockEvent) + require.NoError(t, err) + + eventsQueryClient := testeventsquery.NewAnyTimesEventsBytesEventsQueryClient( + ctx, t, + committedBlocksQuery, + expectedEventBz, + ) + deps := depinject.Supply(eventsQueryClient) // Set up block client. @@ -58,60 +57,54 @@ func TestBlockClient(t *testing.T) { require.NoError(t, err) require.NotNil(t, blockClient) - // Run LatestBlock and CommittedBlockSequence concurrently because they can - // block, leading to an unresponsive test. This function sends multiple values - // on the actualBlockCh which are all asserted against in blockAssertionLoop. - // If any of the methods under test hang, the test will time out. - var ( - actualBlockCh = make(chan client.Block, 1) - done = make(chan struct{}, 1) - ) - go func() { - // Test LatestBlock method. - actualBlock := blockClient.LatestBlock(ctx) - require.Equal(t, expectedHeight, actualBlock.Height()) - require.Equal(t, expectedHash, actualBlock.Hash()) - - // Test CommittedBlockSequence method. - blockObservable := blockClient.CommittedBlocksSequence(ctx) - require.NotNil(t, blockObservable) - - // Ensure that the observable is replayable via Last. - actualBlockCh <- blockObservable.Last(ctx, 1)[0] - - // Ensure that the observable is replayable via Subscribe. - blockObserver := blockObservable.Subscribe(ctx) - for block := range blockObserver.Ch() { - actualBlockCh <- block - break - } - - // Signal test completion - done <- struct{}{} - }() - - // blockAssertionLoop ensures that the blocks retrieved from both LatestBlock - // method and CommittedBlocksSequence method match the expected block height - // and hash. This loop waits for blocks to be sent on the actualBlockCh channel - // by the methods being tested. Once the methods are done, they send a signal on - // the "done" channel. If the blockAssertionLoop doesn't receive any block or - // the done signal within a specific timeout, it assumes something has gone wrong - // and fails the test. -blockAssertionLoop: - for { - select { - case actualBlock := <-actualBlockCh: - require.Equal(t, expectedHeight, actualBlock.Height()) - require.Equal(t, expectedHash, actualBlock.Hash()) - case <-done: - break blockAssertionLoop - case <-time.After(blockAssertionLoopTimeout): - t.Fatal("timed out waiting for block event") - } + tests := []struct { + name string + fn func() client.Block + }{ + { + name: "LatestBlock successfully returns latest block", + fn: func() client.Block { + lastBlock := blockClient.LatestBlock(ctx) + return lastBlock + }, + }, + { + name: "CommittedBlocksSequence successfully returns latest block", + fn: func() client.Block { + blockObservable := blockClient.CommittedBlocksSequence(ctx) + require.NotNil(t, blockObservable) + + // Ensure that the observable is replayable via Last. + lastBlock := blockObservable.Last(ctx, 1)[0] + require.Equal(t, expectedHeight, lastBlock.Height()) + require.Equal(t, expectedHash, lastBlock.Hash()) + + return lastBlock + }, + }, } - // Wait a tick for the observables to be set up. - time.Sleep(time.Millisecond) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var actualBlockCh = make(chan client.Block, 10) + + // Run test functions asynchronously because they can block, leading + // to an unresponsive test. If any of the methods under test hang, + // the test will time out in the select statement that follows. + go func(fn func() client.Block) { + actualBlockCh <- fn() + close(actualBlockCh) + }(tt.fn) + + select { + case actualBlock := <-actualBlockCh: + require.Equal(t, expectedHeight, actualBlock.Height()) + require.Equal(t, expectedHash, actualBlock.Hash()) + case <-time.After(testTimeoutDuration): + t.Fatal("timed out waiting for block event") + } + }) + } blockClient.Close() } diff --git a/x/application/client/cli/tx_delegate_to_gateway.go b/x/application/client/cli/tx_delegate_to_gateway.go index 324f88622..ea251e6cd 100644 --- a/x/application/client/cli/tx_delegate_to_gateway.go +++ b/x/application/client/cli/tx_delegate_to_gateway.go @@ -22,7 +22,7 @@ that delegates authority to the gateway specified to sign relays requests for th act on the behalf of the application during a session. Example: -$ pocketd --home=$(POCKETD_HOME) tx application delegate-to-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx application delegate-to-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { gatewayAddress := args[0] diff --git a/x/application/client/cli/tx_stake_application.go b/x/application/client/cli/tx_stake_application.go index 4b077e6c2..510cfd648 100644 --- a/x/application/client/cli/tx_stake_application.go +++ b/x/application/client/cli/tx_stake_application.go @@ -27,7 +27,7 @@ func CmdStakeApplication() *cobra.Command { will stake the tokens and serviceIds and associate them with the application specified by the 'from' address. Example: -$ pocketd --home=$(POCKETD_HOME) tx application stake-application 1000upokt svc1,svc2,svc3 --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx application stake-application 1000upokt svc1,svc2,svc3 --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) (err error) { stakeString := args[0] diff --git a/x/application/client/cli/tx_undelegate_from_gateway.go b/x/application/client/cli/tx_undelegate_from_gateway.go index 95a770baa..308a5d8a0 100644 --- a/x/application/client/cli/tx_undelegate_from_gateway.go +++ b/x/application/client/cli/tx_undelegate_from_gateway.go @@ -22,7 +22,7 @@ that removes the authority from the gateway specified to sign relays requests fo act on the behalf of the application during a session. Example: -$ pocketd --home=$(POCKETD_HOME) tx application undelegate-from-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx application undelegate-from-gateway $(GATEWAY_ADDR) --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { gatewayAddress := args[0] diff --git a/x/application/client/cli/tx_unstake_application.go b/x/application/client/cli/tx_unstake_application.go index ebf720a82..bfbf10e32 100644 --- a/x/application/client/cli/tx_unstake_application.go +++ b/x/application/client/cli/tx_unstake_application.go @@ -22,7 +22,7 @@ func CmdUnstakeApplication() *cobra.Command { the application specified by the 'from' address. Example: -$ pocketd --home=$(POCKETD_HOME) tx application unstake-application --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx application unstake-application --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) (err error) { diff --git a/x/gateway/client/cli/tx_stake_gateway.go b/x/gateway/client/cli/tx_stake_gateway.go index 2104b2523..2c363b43b 100644 --- a/x/gateway/client/cli/tx_stake_gateway.go +++ b/x/gateway/client/cli/tx_stake_gateway.go @@ -21,7 +21,7 @@ func CmdStakeGateway() *cobra.Command { Long: `Stake a gateway with the provided parameters. This is a broadcast operation that will stake the tokens and associate them with the gateway specified by the 'from' address. Example: -$ pocketd --home=$(POCKETD_HOME) tx gateway stake-gateway 1000upokt --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx gateway stake-gateway 1000upokt --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) (err error) { clientCtx, err := client.GetClientTxContext(cmd) diff --git a/x/gateway/client/cli/tx_unstake_gateway.go b/x/gateway/client/cli/tx_unstake_gateway.go index b57fd9eb7..e417b7540 100644 --- a/x/gateway/client/cli/tx_unstake_gateway.go +++ b/x/gateway/client/cli/tx_unstake_gateway.go @@ -21,7 +21,7 @@ func CmdUnstakeGateway() *cobra.Command { Long: `Unstake a gateway. This is a broadcast operation that will unstake the gateway specified by the 'from' address. Example: -$ pocketd --home=$(POCKETD_HOME) tx gateway unstake-gateway --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx gateway unstake-gateway --keyring-backend test --from $(GATEWAY) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, _ []string) (err error) { clientCtx, err := client.GetClientTxContext(cmd) diff --git a/x/supplier/client/cli/tx_stake_supplier.go b/x/supplier/client/cli/tx_stake_supplier.go index f223799cf..eac4b4044 100644 --- a/x/supplier/client/cli/tx_stake_supplier.go +++ b/x/supplier/client/cli/tx_stake_supplier.go @@ -32,7 +32,7 @@ of comma separated values of the form 'service;url' where 'service' is the servi For example, an application that stakes for 'anvil' could be matched with a supplier staking for 'anvil;http://anvil:8547'. Example: -$ pocketd --home=$(POCKETD_HOME) tx supplier stake-supplier 1000upokt anvil;http://anvil:8547 --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx supplier stake-supplier 1000upokt anvil;http://anvil:8547 --keyring-backend test --from $(APP) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) (err error) { stakeString := args[0] diff --git a/x/supplier/client/cli/tx_unstake_supplier.go b/x/supplier/client/cli/tx_unstake_supplier.go index 40ac4a83f..2daf7c00a 100644 --- a/x/supplier/client/cli/tx_unstake_supplier.go +++ b/x/supplier/client/cli/tx_unstake_supplier.go @@ -17,7 +17,7 @@ func CmdUnstakeSupplier() *cobra.Command { Long: `Unstake an supplier with the provided parameters. This is a broadcast operation that will unstake the supplier specified by the 'from' address. Example: -$ pocketd --home=$(POCKETD_HOME) tx supplier unstake-supplier --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE)`, +$ poktrolld --home=$(POCKETD_HOME) tx supplier unstake-supplier --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE)`, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) (err error) { From aab9e17634fd82431baafeb68abec8633cb52ead Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 7 Nov 2023 02:54:25 +0100 Subject: [PATCH 04/32] fix: remove signature field before signing --- pkg/appserver/relay_verifier.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/appserver/relay_verifier.go b/pkg/appserver/relay_verifier.go index 1c3b2ada5..ec7992640 100644 --- a/pkg/appserver/relay_verifier.go +++ b/pkg/appserver/relay_verifier.go @@ -21,6 +21,9 @@ func (app *appServer) verifyResponse( return err } + signature := relayResponse.Meta.SupplierSignature + relayResponse.Meta.SupplierSignature = nil + // Marshal the relay response payload to bytes and get the hash. var payloadBz []byte if _, err = relayResponse.Payload.MarshalTo(payloadBz); err != nil { @@ -29,7 +32,7 @@ func (app *appServer) verifyResponse( hash := crypto.Sha256(payloadBz) // Verify the relay response signature. - if !pubKey.VerifySignature(hash, relayResponse.Meta.SupplierSignature) { + if !pubKey.VerifySignature(hash, signature) { return ErrInvalidRelayResponseSignature } From 0f2a53f3976180d58043a32e1560922e4862771f Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:10:38 +0000 Subject: [PATCH 05/32] chore: go.mod --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 95c298124..4b5bdd434 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( cosmossdk.io/math v1.0.1 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 - github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -27,7 +26,6 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -71,6 +69,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -266,6 +265,7 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect From 6e265c28d014bf4d8f08766bbb6b2585f9b6a268 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:52:16 +0000 Subject: [PATCH 06/32] feat: add ring signatures --- go.mod | 6 +- go.sum | 4 + .../endpoint_selector.go | 4 +- pkg/{appserver => appgateserver}/errors.go | 3 +- pkg/{appserver => appgateserver}/jsonrpc.go | 39 ++++-- .../relay_verifier.go | 17 +-- pkg/appgateserver/rings.go | 121 ++++++++++++++++++ pkg/{appserver => appgateserver}/server.go | 105 +++++++++------ pkg/{appserver => appgateserver}/session.go | 8 +- pkg/relayer/proxy/errors.go | 7 +- pkg/relayer/proxy/interface.go | 3 +- pkg/relayer/proxy/proxy.go | 78 ++++++++++- pkg/relayer/proxy/rings.go | 99 ++++++++++++++ pkg/signer/interface.go | 9 ++ pkg/signer/ring_signer.go | 33 +++++ pkg/signer/simple_signer.go | 23 ++++ x/service/types/relay.go | 21 +++ 17 files changed, 507 insertions(+), 73 deletions(-) rename pkg/{appserver => appgateserver}/endpoint_selector.go (95%) rename pkg/{appserver => appgateserver}/errors.go (72%) rename pkg/{appserver => appgateserver}/jsonrpc.go (80%) rename pkg/{appserver => appgateserver}/relay_verifier.go (82%) create mode 100644 pkg/appgateserver/rings.go rename pkg/{appserver => appgateserver}/server.go (61%) rename pkg/{appserver => appgateserver}/session.go (90%) create mode 100644 pkg/relayer/proxy/rings.go create mode 100644 pkg/signer/interface.go create mode 100644 pkg/signer/ring_signer.go create mode 100644 pkg/signer/simple_signer.go create mode 100644 x/service/types/relay.go diff --git a/go.mod b/go.mod index a8ef04265..806935020 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,10 @@ require ( cosmossdk.io/depinject v1.0.0-alpha.3 cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.1 + github.com/athanorlabs/go-dleq v0.1.0 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 + github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -18,6 +20,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 + github.com/noot/ring-go v0.0.0-20231019173746-6c4b33bcf03f github.com/pokt-network/smt v0.7.1 github.com/regen-network/gocuke v0.6.2 github.com/spf13/cast v1.5.1 @@ -27,6 +30,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -70,7 +74,6 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect - github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -266,7 +269,6 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 64f7566f0..9c6eb64d5 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/ashanbrown/forbidigo v1.3.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBFg8t0sG2FIxmI= github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI= +github.com/athanorlabs/go-dleq v0.1.0 h1:0/llWZG8fz2uintMBKOiBC502zCsDA8nt8vxI73W9Qc= +github.com/athanorlabs/go-dleq v0.1.0/go.mod h1:DWry6jSD7A13MKmeZA0AX3/xBeQCXDoygX99VPwL3yU= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -1478,6 +1480,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nishanths/exhaustive v0.8.1/go.mod h1:qj+zJJUgJ76tR92+25+03oYUhzF4R7/2Wk7fGTfCHmg= github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/noot/ring-go v0.0.0-20231019173746-6c4b33bcf03f h1:1+NP/H13eFAqBYrGpRkbJUWVWIO2Zr2eP7a/q0UtZVQ= +github.com/noot/ring-go v0.0.0-20231019173746-6c4b33bcf03f/go.mod h1:0t3gzoSfW2bkTce1E/Jis3MQpjiKGhAgqieFK+nkQsI= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= diff --git a/pkg/appserver/endpoint_selector.go b/pkg/appgateserver/endpoint_selector.go similarity index 95% rename from pkg/appserver/endpoint_selector.go rename to pkg/appgateserver/endpoint_selector.go index aff1e916e..37b58e3e8 100644 --- a/pkg/appserver/endpoint_selector.go +++ b/pkg/appgateserver/endpoint_selector.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import ( "context" @@ -13,7 +13,7 @@ import ( // first available URL but future optimizations can be introduced. // TODO(@h5law): Look into different endpoint selection depending on their suitability. // getRelayerUrl gets the URL of the relayer for the given service. -func (app *appServer) getRelayerUrl( +func (app *appGateServer) getRelayerUrl( ctx context.Context, serviceId string, rpcType sharedtypes.RPCType, diff --git a/pkg/appserver/errors.go b/pkg/appgateserver/errors.go similarity index 72% rename from pkg/appserver/errors.go rename to pkg/appgateserver/errors.go index 722c6aaf8..7209f02c5 100644 --- a/pkg/appserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import sdkerrors "cosmossdk.io/errors" @@ -6,4 +6,5 @@ var ( codespace = "appclient" ErrInvalidRelayResponseSignature = sdkerrors.Register(codespace, 1, "invalid relay response signature") ErrNoRelayEndpoints = sdkerrors.Register(codespace, 2, "no relay endpoints found") + ErrInvalidRequestURL = sdkerrors.Register(codespace, 3, "invalid request URL") ) diff --git a/pkg/appserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go similarity index 80% rename from pkg/appserver/jsonrpc.go rename to pkg/appgateserver/jsonrpc.go index 0f4370300..548ddb559 100644 --- a/pkg/appserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import ( "bytes" @@ -13,9 +13,9 @@ import ( ) // handleJSONRPCRelay handles JSON RPC relay requests. -func (app *appServer) handleJSONRPCRelay( +func (app *appGateServer) handleJSONRPCRelay( ctx context.Context, - serviceId string, + appAddress, serviceId string, request *http.Request, writer http.ResponseWriter, ) error { @@ -25,18 +25,11 @@ func (app *appServer) handleJSONRPCRelay( return err } - // Hash and sign the request payload. - hash := crypto.Sha256(payloadBz) - signature, _, err := app.keyring.Sign(app.keyName, hash) - if err != nil { - return err - } - // Create the relay request payload. relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) - session, err := app.getCurrentSession(ctx, serviceId) + session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { return err } @@ -50,12 +43,32 @@ func (app *appServer) handleJSONRPCRelay( // Create the relay request. relayRequest := &types.RelayRequest{ Meta: &types.RelayRequestMetadata{ - SessionHeader: session.Header, - Signature: signature, + // SessionHeader: session.Header, + Signature: nil, }, Payload: relayRequestPayload, } + // Get the application's signer. + signer, err := app.getRingSingerForAppAddress(ctx, appAddress) + if err != nil { + return err + } + + // Hash and sign the request's signable bytes. + signableBz, err := relayRequest.GetSignableBytes() + if err != nil { + return err + } + hash := crypto.Sha256(signableBz) + var hash32 [32]byte + copy(hash32[:], hash) + signature, err := signer.Sign(hash32) + if err != nil { + return err + } + relayRequest.Meta.Signature = signature + // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := relayRequest.Marshal() if err != nil { diff --git a/pkg/appserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go similarity index 82% rename from pkg/appserver/relay_verifier.go rename to pkg/appgateserver/relay_verifier.go index ec7992640..d942a3704 100644 --- a/pkg/appserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import ( "context" @@ -11,25 +11,26 @@ import ( ) // verifyResponse verifies the relay response signature. -func (app *appServer) verifyResponse( +func (app *appGateServer) verifyResponse( ctx context.Context, supplierAddress string, relayResponse *types.RelayResponse, ) error { + // Get the supplier's public key. pubKey, err := app.getSupplierPubKeyFromAddress(ctx, supplierAddress) if err != nil { return err } + // Extract the supplier's signature signature := relayResponse.Meta.SupplierSignature - relayResponse.Meta.SupplierSignature = nil - // Marshal the relay response payload to bytes and get the hash. - var payloadBz []byte - if _, err = relayResponse.Payload.MarshalTo(payloadBz); err != nil { + // Get the relay response signable bytes and hash them. + responseBz, err := relayResponse.GetSignableBytes() + if err != nil { return err } - hash := crypto.Sha256(payloadBz) + hash := crypto.Sha256(responseBz) // Verify the relay response signature. if !pubKey.VerifySignature(hash, signature) { @@ -42,7 +43,7 @@ func (app *appServer) verifyResponse( // getSupplierPubKeyFromAddress gets the supplier's public key from the cache or queries // if it is not found. // The public key is then cached before being returned. -func (app *appServer) getSupplierPubKeyFromAddress( +func (app *appGateServer) getSupplierPubKeyFromAddress( ctx context.Context, supplierAddress string, ) (cryptotypes.PubKey, error) { diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go new file mode 100644 index 000000000..2661945ce --- /dev/null +++ b/pkg/appgateserver/rings.go @@ -0,0 +1,121 @@ +// TODO(@h5law): Move all this logic out into a shared package +package appgateserver + +import ( + "context" + "fmt" + + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" + ringtypes "github.com/athanorlabs/go-dleq/types" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + ring "github.com/noot/ring-go" + "github.com/pokt-network/poktroll/pkg/signer" + apptypes "github.com/pokt-network/poktroll/x/application/types" +) + +// getRingSingerForAppAddress returns the RingSinger used to sign relays. +// This method first attempts to get the points of the ring from the cache, if it +// fails it queries the application module for the points and creates the ring. +func (app *appGateServer) getRingSingerForAppAddress(ctx context.Context, appAddress string) (*signer.RingSigner, error) { + var ring *ring.Ring + var err error + + // lock the cache for reading + app.ringCacheMutex.RLock() + defer app.ringCacheMutex.RUnlock() + + // check if the ring is in the cache + points, ok := app.ringCache[appAddress] + if !ok { + // if the ring is not in the cache, get it from the application module + ring, err = app.getRingForAppAddress(ctx, appAddress) + } else { + // if the ring is in the cache, create it from the points + ring, err = newRingFromPoints(points) + } + if err != nil { + return nil, err + } + + // return the ring signer + return signer.NewRingSigner(ring, app.signingKey), nil +} + +// getRingForAppAddress returns the RingSinger used to sign relays. It does so by fetching +// the latest information from the application module and creating the correct ring. +// This method also caches the ring's public keys for future use. +func (app *appGateServer) getRingForAppAddress(ctx context.Context, appAddress string) (*ring.Ring, error) { + points, err := app.getDelegatedPubKeysForAddress(ctx, appAddress) + if err != nil { + return nil, err + } + return newRingFromPoints(points) +} + +// newRingFromPoints creates a new ring from a slice of points on the secp256k1 curve +func newRingFromPoints(points []ringtypes.Point) (*ring.Ring, error) { + return ring.NewFixedKeyRingFromPublicKeys(ring_secp256k1.NewCurve(), points) +} + +// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given application +// address, by querying the portal module for it's delegated pubkeys +func (app *appGateServer) getDelegatedPubKeysForAddress( + ctx context.Context, + appAddress string, +) ([]ringtypes.Point, error) { + // get the application's on chain state + req := &apptypes.QueryGetApplicationRequest{Address: appAddress} + res, err := app.applicationQuerier.Application(ctx, req) + if err != nil { + return nil, fmt.Errorf("unable to retrieve application for address: %s [%w]", appAddress, err) + } + + // create a slice of addresses for the ring + ringAddresses := make([]string, len(res.Application.DelegateeGatewayAddresses)+1) // +1 for app address + ringAddresses[0] = appAddress // app address is index 0 + copy(ringAddresses[1:], res.Application.DelegateeGatewayAddresses) // copy the gateway addresses + + // get the points on the secp256k1 curve for the addresses + points, err := app.addressesToPoints(ctx, ringAddresses) + if err != nil { + return nil, err + } + + // update the cache overwriting the previous value + app.ringCacheMutex.Lock() + defer app.ringCacheMutex.Unlock() + app.ringCache[appAddress] = points + + // return the public key points on the secp256k1 curve + return points, nil +} + +// addressesToPoints converts a slice of addresses to a slice of points on the secp256k1 curve +// it does so by querying the account module for the public key for each address and converting +// them to the corresponding points on the secp256k1 curve +func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { + curve := ring_secp256k1.NewCurve() + points := make([]ringtypes.Point, len(addresses)) + for i, addr := range addresses { + pubKeyReq := &accounttypes.QueryAccountRequest{Address: addr} + pubKeyRes, err := app.accountQuerier.Account(ctx, pubKeyReq) + if err != nil { + return nil, fmt.Errorf("unable to get account for address: %s [%w]", addr, err) + } + acc := new(accounttypes.BaseAccount) + if err := acc.Unmarshal(pubKeyRes.Account.Value); err != nil { + return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) + } + key := acc.GetPubKey() + if _, ok := key.(*secp256k1.PubKey); !ok { + return nil, fmt.Errorf("public key is not a secp256k1 key: got %T", key) + } + point, err := curve.DecodeToPoint(key.Bytes()) + if err != nil { + return nil, err + } + points[i] = point + } + return points, nil +} diff --git a/pkg/appserver/server.go b/pkg/appgateserver/server.go similarity index 61% rename from pkg/appserver/server.go rename to pkg/appgateserver/server.go index 88a9e64b7..0c995a418 100644 --- a/pkg/appserver/server.go +++ b/pkg/appgateserver/server.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import ( "context" @@ -8,33 +8,44 @@ import ( "strings" "sync" + sdkerrors "cosmossdk.io/errors" + ringtypes "github.com/athanorlabs/go-dleq/types" sdkclient "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/crypto/keyring" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" blocktypes "github.com/pokt-network/poktroll/pkg/client" + apptypes "github.com/pokt-network/poktroll/x/application/types" "github.com/pokt-network/poktroll/x/service/types" sessiontypes "github.com/pokt-network/poktroll/x/session/types" ) -// appServer is the server that listens for application requests and relays them to the supplier. +// appGateServer is the server that listens for application requests and relays them to the supplier. // it is responsible for maintaining the current session for the application, signing the requests, // and verifying the response signatures. -type appServer struct { - // TODO(@h5law): Replace with ring signature. - // keyName is the name of the key in the keyring that will be used to sign relay requests. - keyName string - keyring keyring.Keyring +// The appGateServer is the basis for both applications and gateways, depending on whether the application +// is running their own instance of the appGateServer or they are sending requests to a gateway running an +// instance of the appGateServer, they will need to either include the application address in the request or not. +type appGateServer struct { + // signingKey is the scalar point on the appropriate curve corresponding to the + // signer's private key, and is used to sign relay requests via a ring signature + signingKey ringtypes.Scalar + + // ringCache is a cache of the public keys used to create the ring for a given application + // they are stored in a map of application address to a slice of points on the secp256k1 curve + // TODO(@h5law): subscribe to on-chain events to update this cache as the ring changes over time + ringCache map[string][]ringtypes.Point + ringCacheMutex *sync.RWMutex + + // appAddress is the address of the application that the server is serving if + // it is nil then the application address must be included in each request + appAddress string // clientCtx is the client context for the application. // It is used to query for the application's account to unmarshal the supplier's account // and get the public key to verify the relay response signature. clientCtx sdkclient.Context - // appAddress is the address of the application that this app server is running for. - appAddress string - // sessionQuerier is the querier for the session module. // It used to get the current session for the application given a requested service. sessionQuerier sessiontypes.QueryClient @@ -50,6 +61,10 @@ type appServer struct { // It is used to get the the supplier's public key to verify the relay response signature. accountQuerier accounttypes.QueryClient + // applicationQuerier is the querier for the application module. + // It is used to get the ring for a given application address. + applicationQuerier apptypes.QueryClient + // blockClient is the client for the block module. // It is used to get the current block height to query for the current session. blockClient blocktypes.BlockClient @@ -65,20 +80,24 @@ type appServer struct { func NewAppServer( clientCtx sdkclient.Context, - keyName string, - keyring keyring.Keyring, + signKey ringtypes.Scalar, + appAddress string, applicationEndpoint *url.URL, blockClient blocktypes.BlockClient, -) *appServer { +) *appGateServer { sessionQuerier := sessiontypes.NewQueryClient(clientCtx) accountQuerier := accounttypes.NewQueryClient(clientCtx) + applicationQuerier := apptypes.NewQueryClient(clientCtx) - return &appServer{ + return &appGateServer{ + signingKey: signKey, + ringCacheMutex: &sync.RWMutex{}, + ringCache: make(map[string][]ringtypes.Point), + appAddress: appAddress, clientCtx: clientCtx, - keyName: keyName, - keyring: keyring, sessionQuerier: sessionQuerier, accountQuerier: accountQuerier, + applicationQuerier: applicationQuerier, blockClient: blockClient, server: &http.Server{Addr: applicationEndpoint.Host}, supplierAccountCache: make(map[string]cryptotypes.PubKey), @@ -87,20 +106,7 @@ func NewAppServer( // Start starts the application server and blocks until the context is done // or the server returns an error. -func (app *appServer) Start(ctx context.Context) error { - // Get and populate the application address from the keyring. - keyRecord, err := app.keyring.Key(app.keyName) - if err != nil { - return err - } - - accAddress, err := keyRecord.GetAddress() - if err != nil { - return err - } - - app.appAddress = accAddress.String() - +func (app *appGateServer) Start(ctx context.Context) error { // Shutdown the HTTP server when the context is done. go func() { <-ctx.Done() @@ -112,7 +118,7 @@ func (app *appServer) Start(ctx context.Context) error { } // Stop stops the application server and returns any error that occurred. -func (app *appServer) Stop(ctx context.Context) error { +func (app *appGateServer) Stop(ctx context.Context) error { return app.server.Shutdown(ctx) } @@ -121,20 +127,47 @@ func (app *appServer) Stop(ctx context.Context) error { // After receiving the response from the supplier, it verifies the response signature // before returning the response to the application. // The serviceId is extracted from the request path. -// The request's path should be of the form "/{serviceId}/[other/path/segments]", +// The request's path should be of the form: +// +// "://host:port/serviceId[/other/path/segments]?senderAddr=" +// // where the serviceId is the id of the service that the application is requesting // and the other (possible) path segments are the JSON RPC request path. -func (app *appServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { +func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { ctx := request.Context() // Extract the serviceId from the request path. path := request.URL.Path serviceId := strings.Split(path, "/")[1] + appAddress := request.URL.Query().Get("senderAddr") + + // ensure the app address is present + if appAddress == "" && app.appAddress == "" { + app.replyWithError( + writer, + sdkerrors.Wrapf(ErrInvalidRequestURL, "missing sender address query parameter: got %s", request.URL.String()), + ) + log.Print("ERROR: no application address provided") + return + } else if appAddress != "" && appAddress != app.appAddress { + app.replyWithError( + writer, + sdkerrors.Wrapf( + ErrInvalidRequestURL, + "sender address query parameter does not match the application address: got %s, want %s", + appAddress, app.appAddress, + ), + ) + log.Print("ERROR: application address does not match sender address query parameter") + return + } else if appAddress == "" { + appAddress = app.appAddress + } // TODO_TECHDEBT: Currently, there is no information about the RPC type requested. It should // be extracted from the request and used to determine the RPC type to handle. handle*Relay() // calls should be wrapped into a switch statement to handle different types of relays. - err := app.handleJSONRPCRelay(ctx, serviceId, request, writer) + err := app.handleJSONRPCRelay(ctx, appAddress, serviceId, request, writer) if err != nil { // Reply with an error response if there was an error handling the relay. app.replyWithError(writer, err) @@ -148,7 +181,7 @@ func (app *appServer) ServeHTTP(writer http.ResponseWriter, request *http.Reques // replyWithError replies to the application with an error response. // TODO_TECHDEBT: This method should be aware of the nature of the error to use the appropriate JSONRPC // Code, Message and Data. Possibly by augmenting the passed in error with the adequate information. -func (app *appServer) replyWithError(writer http.ResponseWriter, err error) { +func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) { relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ diff --git a/pkg/appserver/session.go b/pkg/appgateserver/session.go similarity index 90% rename from pkg/appserver/session.go rename to pkg/appgateserver/session.go index c3b64782f..89b6411c4 100644 --- a/pkg/appserver/session.go +++ b/pkg/appgateserver/session.go @@ -1,4 +1,4 @@ -package appserver +package appgateserver import ( "context" @@ -10,9 +10,9 @@ import ( // getCurrentSession gets the current session for the given service // It returns the current session if it exists and is still valid, otherwise it // queries for the latest session, caches and returns it. -func (app *appServer) getCurrentSession( +func (app *appGateServer) getCurrentSession( ctx context.Context, - serviceId string, + appAddress, serviceId string, ) (*sessiontypes.Session, error) { app.sessionMu.RLock() defer app.sessionMu.RUnlock() @@ -29,7 +29,7 @@ func (app *appServer) getCurrentSession( // Query for the current session. sessionQueryReq := sessiontypes.QueryGetSessionRequest{ - ApplicationAddress: app.appAddress, + ApplicationAddress: appAddress, ServiceId: &sharedtypes.ServiceId{Id: serviceId}, BlockHeight: latestBlock.Height(), } diff --git a/pkg/relayer/proxy/errors.go b/pkg/relayer/proxy/errors.go index 1aa42ab7e..80c1e12bf 100644 --- a/pkg/relayer/proxy/errors.go +++ b/pkg/relayer/proxy/errors.go @@ -3,6 +3,9 @@ package proxy import sdkerrors "cosmossdk.io/errors" var ( - codespace = "relayer/proxy" - ErrUnsupportedRPCType = sdkerrors.Register(codespace, 1, "unsupported rpc type") + codespace = "relayer/proxy" + ErrUnsupportedRPCType = sdkerrors.Register(codespace, 1, "unsupported rpc type") + ErrInvalidRelayRequest = sdkerrors.Register(codespace, 2, "invalid relay request") + ErrInvalidRequestSignature = sdkerrors.Register(codespace, 3, "invalid relay request signature") + ErrInvalidRelayResponse = sdkerrors.Register(codespace, 4, "invalid relay response") ) diff --git a/pkg/relayer/proxy/interface.go b/pkg/relayer/proxy/interface.go index 27ee83e72..c84ff756c 100644 --- a/pkg/relayer/proxy/interface.go +++ b/pkg/relayer/proxy/interface.go @@ -13,7 +13,6 @@ import ( // While handling requests and responding in a closed loop, it also notifies // the miner about the relays that have been served. type RelayerProxy interface { - // Start starts all advertised relay servers and returns an error if any of them fail to start. Start(ctx context.Context) error @@ -27,7 +26,7 @@ type RelayerProxy interface { // VerifyRelayRequest is a shared method used by RelayServers to check the // relay request signature and session validity. - VerifyRelayRequest(relayRequest *types.RelayRequest) (isValid bool, err error) + VerifyRelayRequest(ctx context.Context, relayRequest *types.RelayRequest) (isValid bool, err error) // SignRelayResponse is a shared method used by RelayServers to sign the relay response. SignRelayResponse(relayResponse *types.RelayResponse) ([]byte, error) diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 033e9caaf..147d88824 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -3,16 +3,23 @@ package proxy import ( "context" "net/url" + "sync" + sdkerrors "cosmossdk.io/errors" + ringtypes "github.com/athanorlabs/go-dleq/types" + "github.com/cometbft/cometbft/crypto" sdkclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/crypto/keyring" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + "github.com/noot/ring-go" "golang.org/x/sync/errgroup" // TODO_INCOMPLETE(@red-0ne): Import the appropriate block client interface once available. // blocktypes "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/signer" + apptypes "github.com/pokt-network/poktroll/x/application/types" "github.com/pokt-network/poktroll/x/service/types" sessiontypes "github.com/pokt-network/poktroll/x/session/types" suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" @@ -49,6 +56,10 @@ type relayerProxy struct { // which is needed to check if the relay proxy should be serving an incoming relay request. sessionQuerier sessiontypes.QueryClient + // applicationQuerier is the querier for the application module. + // It is used to get the ring for a given application address. + applicationQuerier apptypes.QueryClient + // advertisedRelayServers is a map of the services provided by the relayer proxy. Each provided service // has the necessary information to start the server that listens for incoming relay requests and // the client that relays the request to the supported proxied service. @@ -63,6 +74,12 @@ type relayerProxy struct { // servedRelaysProducer is a channel that emits the relays that have been served so that the // servedRelays observable can fan out the notifications to its subscribers. servedRelaysProducer chan<- *types.Relay + + // ringCache is a cache of the public keys used to create the ring for a given application + // they are stored in a map of application address to a slice of points on the secp256k1 curve + // TODO(@h5law): subscribe to on-chain events to update this cache as the ring changes over time + ringCache map[string][]ringtypes.Point + ringCacheMutex *sync.RWMutex } func NewRelayerProxy( @@ -76,6 +93,7 @@ func NewRelayerProxy( ) RelayerProxy { accountQuerier := accounttypes.NewQueryClient(clientCtx) supplierQuerier := suppliertypes.NewQueryClient(clientCtx) + applicationQuerier := apptypes.NewQueryClient(clientCtx) sessionQuerier := sessiontypes.NewQueryClient(clientCtx) servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() @@ -86,10 +104,13 @@ func NewRelayerProxy( keyring: keyring, accountsQuerier: accountQuerier, supplierQuerier: supplierQuerier, + applicationQuerier: applicationQuerier, sessionQuerier: sessionQuerier, proxiedServicesEndpoints: proxiedServicesEndpoints, servedRelays: servedRelays, servedRelaysProducer: servedRelaysProducer, + ringCacheMutex: &sync.RWMutex{}, + ringCache: make(map[string][]ringtypes.Point), } } @@ -139,11 +160,62 @@ func (rp *relayerProxy) ServedRelays() observable.Observable[*types.Relay] { } // VerifyRelayRequest is a shared method used by RelayServers to check the relay request signature and session validity. -func (rp *relayerProxy) VerifyRelayRequest(relayRequest *types.RelayRequest) (isValid bool, err error) { - panic("TODO: implement relayerProxy.VerifyRelayRequest") +func (rp *relayerProxy) VerifyRelayRequest(ctx context.Context, relayRequest *types.RelayRequest) (isValid bool, err error) { + // extract the relay request's ring signature + signature := relayRequest.Meta.Signature + if signature == nil { + return false, sdkerrors.Wrapf(ErrInvalidRelayRequest, "missing signature from relay request: %v", relayRequest) + } + ringSig := new(ring.RingSig) + if err := ringSig.Deserialize(signature); err != nil { + return false, sdkerrors.Wrapf(ErrInvalidRequestSignature, "error deserializing signature: %v", err) + } + + // get the ring for the application address of the relay request + appAddress := relayRequest.Meta.SessionHeader.ApplicationAddress + appRing, err := rp.getRingForAppAddress(ctx, appAddress) + if err != nil { + return false, sdkerrors.Wrapf( + ErrInvalidRelayRequest, + "error getting ring for application address %s: %v", appAddress, err, + ) + } + + // verify the ring signature against the ring + if !ringSig.Ring().Equals(appRing) { + return false, sdkerrors.Wrapf( + ErrInvalidRequestSignature, + "ring signature does not match ring for application address %s", appAddress, + ) + } + + // get and hash the signable bytes of the relay request + signableBz, err := relayRequest.GetSignableBytes() + if err != nil { + return false, sdkerrors.Wrapf(ErrInvalidRelayRequest, "error getting signable bytes: %v", err) + } + hash := crypto.Sha256(signableBz) + var hash32 [32]byte + copy(hash32[:], hash) + + // verify the relay request's signature + return ringSig.Verify(hash32), nil } // SignRelayResponse is a shared method used by RelayServers to sign the relay response. func (rp *relayerProxy) SignRelayResponse(relayResponse *types.RelayResponse) ([]byte, error) { - panic("TODO: implement relayerProxy.SignRelayResponse") + // create a simple signer for the request + signer := signer.NewSimpleSigner(rp.keyring, rp.keyName) + + // extract and hash the relay response's signable bytes + signableBz, err := relayResponse.GetSignableBytes() + if err != nil { + return nil, sdkerrors.Wrapf(ErrInvalidRelayResponse, "error getting signable bytes: %v", err) + } + hash := crypto.Sha256(signableBz) + var hash32 [32]byte + copy(hash32[:], hash) + + // sign the relay response + return signer.Sign(hash32) } diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go new file mode 100644 index 000000000..2474fac9f --- /dev/null +++ b/pkg/relayer/proxy/rings.go @@ -0,0 +1,99 @@ +// TODO(@h5law): Move all this logic out into a shared package +package proxy + +import ( + "context" + "fmt" + + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" + ringtypes "github.com/athanorlabs/go-dleq/types" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" + ring "github.com/noot/ring-go" + apptypes "github.com/pokt-network/poktroll/x/application/types" +) + +// getRingForAppAddress returns the RingSinger used to sign relays. It does so by fetching +// the latest information from the application module and creating the correct ring. +// This method also caches the ring's public keys for future use. +func (rp *relayerProxy) getRingForAppAddress(ctx context.Context, appAddress string) (*ring.Ring, error) { + // lock the cache for reading + rp.ringCacheMutex.RLock() + defer rp.ringCacheMutex.RUnlock() + + // check if the ring is in the cache + points, ok := rp.ringCache[appAddress] + if !ok { + // if the ring is not in the cache, get it from the application module + return rp.getRingForAppAddress(ctx, appAddress) + } + // if the ring is in the cache, create it from the points + return newRingFromPoints(points) +} + +// newRingFromPoints creates a new ring from a slice of points on the secp256k1 curve +func newRingFromPoints(points []ringtypes.Point) (*ring.Ring, error) { + return ring.NewFixedKeyRingFromPublicKeys(ring_secp256k1.NewCurve(), points) +} + +// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given application +// address, by querying the portal module for it's delegated pubkeys +func (rp *relayerProxy) getDelegatedPubKeysForAddress( + ctx context.Context, + appAddress string, +) ([]ringtypes.Point, error) { + // get the application's on chain state + req := &apptypes.QueryGetApplicationRequest{Address: appAddress} + res, err := rp.applicationQuerier.Application(ctx, req) + if err != nil { + return nil, fmt.Errorf("unable to retrieve application for address: %s [%w]", appAddress, err) + } + + // create a slice of addresses for the ring + ringAddresses := make([]string, len(res.Application.DelegateeGatewayAddresses)+1) // +1 for app address + ringAddresses[0] = appAddress // app address is index 0 + copy(ringAddresses[1:], res.Application.DelegateeGatewayAddresses) // copy the gateway addresses + + // get the points on the secp256k1 curve for the addresses + points, err := rp.addressesToPoints(ctx, ringAddresses) + if err != nil { + return nil, err + } + + // update the cache overwriting the previous value + rp.ringCacheMutex.Lock() + defer rp.ringCacheMutex.Unlock() + rp.ringCache[appAddress] = points + + // return the public key points on the secp256k1 curve + return points, nil +} + +// addressesToPoints converts a slice of addresses to a slice of points on the secp256k1 curve +// it does so by querying the account module for the public key for each address and converting +// them to the corresponding points on the secp256k1 curve +func (rp *relayerProxy) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { + curve := ring_secp256k1.NewCurve() + points := make([]ringtypes.Point, len(addresses)) + for i, addr := range addresses { + pubKeyReq := &accounttypes.QueryAccountRequest{Address: addr} + pubKeyRes, err := rp.accountsQuerier.Account(ctx, pubKeyReq) + if err != nil { + return nil, fmt.Errorf("unable to get account for address: %s [%w]", addr, err) + } + acc := new(accounttypes.BaseAccount) + if err := acc.Unmarshal(pubKeyRes.Account.Value); err != nil { + return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) + } + key := acc.GetPubKey() + if _, ok := key.(*secp256k1.PubKey); !ok { + return nil, fmt.Errorf("public key is not a secp256k1 key: got %T", key) + } + point, err := curve.DecodeToPoint(key.Bytes()) + if err != nil { + return nil, err + } + points[i] = point + } + return points, nil +} diff --git a/pkg/signer/interface.go b/pkg/signer/interface.go new file mode 100644 index 000000000..63b40b86d --- /dev/null +++ b/pkg/signer/interface.go @@ -0,0 +1,9 @@ +package signer + +// Signer is an interface that abstracts the signing of a message, it is used +// to sign both relay requests and responses via one of the two implementations. +// The Signer interface expects a 32 byte message (sha256 hash) and returns a +// byte slice containing the signature or any error that occurred during signing. +type Signer interface { + Sign(msg [32]byte) (signature []byte, err error) +} diff --git a/pkg/signer/ring_signer.go b/pkg/signer/ring_signer.go new file mode 100644 index 000000000..bb6cf549c --- /dev/null +++ b/pkg/signer/ring_signer.go @@ -0,0 +1,33 @@ +package signer + +import ( + "fmt" + + ringtypes "github.com/athanorlabs/go-dleq/types" + ring "github.com/noot/ring-go" +) + +var _ Signer = (*RingSigner)(nil) + +// RingSigner is a signer implementation that uses a ring to sign messages, for +// verification the ring signature must be verified and confirmed to be using +// the expected ring. +type RingSigner struct { + ring *ring.Ring + privKey ringtypes.Scalar +} + +// NewRingSigner creates a new RingSigner instance with the ring and private key provided +func NewRingSigner(ring *ring.Ring, privKey ringtypes.Scalar) *RingSigner { + return &RingSigner{ring: ring, privKey: privKey} +} + +// Sign uses the ring and private key to sign the message provided and returns the +// serialised ring signature that can be deserialised and verified by the verifier +func (r *RingSigner) Sign(msg [32]byte) ([]byte, error) { + ringSig, err := r.ring.Sign(msg, r.privKey) + if err != nil { + return nil, fmt.Errorf("failed to sign message [%v]: %w", msg, err) + } + return ringSig.Serialize() +} diff --git a/pkg/signer/simple_signer.go b/pkg/signer/simple_signer.go new file mode 100644 index 000000000..432c1c8a3 --- /dev/null +++ b/pkg/signer/simple_signer.go @@ -0,0 +1,23 @@ +package signer + +import "github.com/cosmos/cosmos-sdk/crypto/keyring" + +var _ Signer = (*SimpleSigner)(nil) + +// SimpleSigner is a signer implementation that uses the local keyring to sign +// messages, for verification using the signer's corresponding public key +type SimpleSigner struct { + keyring keyring.Keyring + keyName string +} + +// NewSimpleSigner creates a new SimpleSigner instance with the keyring and keyName provided +func NewSimpleSigner(keyring keyring.Keyring, keyName string) *SimpleSigner { + return &SimpleSigner{keyring: keyring, keyName: keyName} +} + +// Sign signs the given message using the SimpleSigner's keyring and keyName +func (s *SimpleSigner) Sign(msg [32]byte) (signature []byte, err error) { + sig, _, err := s.keyring.Sign(s.keyName, msg[:]) + return sig, err +} diff --git a/x/service/types/relay.go b/x/service/types/relay.go new file mode 100644 index 000000000..f9ec52901 --- /dev/null +++ b/x/service/types/relay.go @@ -0,0 +1,21 @@ +package types + +// GetSignableBytes returns the signable bytes for the relay request +// this involves setting the signature to nil and marshalling the message. +// A value receiver is used to avoid overwriting any pre-existing signature +func (req RelayRequest) GetSignableBytes() ([]byte, error) { + // set signature to nil + req.Meta.Signature = nil + // return the marshalled message + return req.Marshal() +} + +// GetSignableBytes returns the signable bytes for the relay response +// this involves setting the signature to nil and marshalling the message. +// A value receiver is used to avoid overwriting any pre-existing signature +func (res RelayResponse) GetSignableBytes() ([]byte, error) { + // set signature to nil + res.Meta.SupplierSignature = nil + // return the marshalled message + return res.Marshal() +} From c1f611594090339508c5ffbbce9d16c26572d34a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:30:11 +0000 Subject: [PATCH 07/32] chore: remove mock files --- pkg/client/gomock_reflect_3526400147/prog.go | 66 -------------------- 1 file changed, 66 deletions(-) delete mode 100644 pkg/client/gomock_reflect_3526400147/prog.go diff --git a/pkg/client/gomock_reflect_3526400147/prog.go b/pkg/client/gomock_reflect_3526400147/prog.go deleted file mode 100644 index 6003ba81a..000000000 --- a/pkg/client/gomock_reflect_3526400147/prog.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "encoding/gob" - "flag" - "fmt" - "os" - "path" - "reflect" - - "github.com/golang/mock/mockgen/model" - - pkg_ "github.com/pokt-network/poktroll/pkg/client" -) - -var output = flag.String("output", "", "The output file name, or empty to use stdout.") - -func main() { - flag.Parse() - - its := []struct { - sym string - typ reflect.Type - }{ - - {"TxContext", reflect.TypeOf((*pkg_.TxContext)(nil)).Elem()}, - - {"TxClient", reflect.TypeOf((*pkg_.TxClient)(nil)).Elem()}, - } - pkg := &model.Package{ - // NOTE: This behaves contrary to documented behaviour if the - // package name is not the final component of the import path. - // The reflect package doesn't expose the package name, though. - Name: path.Base("github.com/pokt-network/poktroll/pkg/client"), - } - - for _, it := range its { - intf, err := model.InterfaceFromInterfaceType(it.typ) - if err != nil { - fmt.Fprintf(os.Stderr, "Reflection: %v\n", err) - os.Exit(1) - } - intf.Name = it.sym - pkg.Interfaces = append(pkg.Interfaces, intf) - } - - outfile := os.Stdout - if len(*output) != 0 { - var err error - outfile, err = os.Create(*output) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to open output file %q", *output) - } - defer func() { - if err := outfile.Close(); err != nil { - fmt.Fprintf(os.Stderr, "failed to close output file %q", *output) - os.Exit(1) - } - }() - } - - if err := gob.NewEncoder(outfile).Encode(pkg); err != nil { - fmt.Fprintf(os.Stderr, "gob encode: %v\n", err) - os.Exit(1) - } -} From c6814849daf5d9f8938ab310d71a5782549d7153 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:32:05 +0000 Subject: [PATCH 08/32] chore: fix spelling errors --- x/service/types/relay.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/service/types/relay.go b/x/service/types/relay.go index f9ec52901..b38d63265 100644 --- a/x/service/types/relay.go +++ b/x/service/types/relay.go @@ -6,7 +6,7 @@ package types func (req RelayRequest) GetSignableBytes() ([]byte, error) { // set signature to nil req.Meta.Signature = nil - // return the marshalled message + // return the marshaled message return req.Marshal() } @@ -16,6 +16,6 @@ func (req RelayRequest) GetSignableBytes() ([]byte, error) { func (res RelayResponse) GetSignableBytes() ([]byte, error) { // set signature to nil res.Meta.SupplierSignature = nil - // return the marshalled message + // return the marshaled message return res.Marshal() } From c787d807f44ee012d473b358e12d90f201d6c668 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:32:56 +0000 Subject: [PATCH 09/32] fixup: spelling mistake --- x/service/types/relay.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/service/types/relay.go b/x/service/types/relay.go index b38d63265..9c0c4809f 100644 --- a/x/service/types/relay.go +++ b/x/service/types/relay.go @@ -1,7 +1,7 @@ package types // GetSignableBytes returns the signable bytes for the relay request -// this involves setting the signature to nil and marshalling the message. +// this involves setting the signature to nil and marshaling the message. // A value receiver is used to avoid overwriting any pre-existing signature func (req RelayRequest) GetSignableBytes() ([]byte, error) { // set signature to nil @@ -11,7 +11,7 @@ func (req RelayRequest) GetSignableBytes() ([]byte, error) { } // GetSignableBytes returns the signable bytes for the relay response -// this involves setting the signature to nil and marshalling the message. +// this involves setting the signature to nil and marshaling the message. // A value receiver is used to avoid overwriting any pre-existing signature func (res RelayResponse) GetSignableBytes() ([]byte, error) { // set signature to nil From 65e9bee6da974aef6613b72c7c379c1a11edd39a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:26:17 +0000 Subject: [PATCH 10/32] feat: add command to start the appgateserver --- cmd/pocketd/cmd/root.go | 6 ++ docs/static/openapi.yml | 30 +++--- go.mod | 4 +- pkg/appgateserver/cmd/cmd.go | 201 +++++++++++++++++++++++++++++++++++ pkg/appgateserver/rings.go | 10 +- pkg/appgateserver/server.go | 12 ++- pkg/client/block/client.go | 7 +- 7 files changed, 243 insertions(+), 27 deletions(-) create mode 100644 pkg/appgateserver/cmd/cmd.go diff --git a/cmd/pocketd/cmd/root.go b/cmd/pocketd/cmd/root.go index 5b1fb3276..cf58f2447 100644 --- a/cmd/pocketd/cmd/root.go +++ b/cmd/pocketd/cmd/root.go @@ -43,6 +43,7 @@ import ( "github.com/pokt-network/poktroll/app" appparams "github.com/pokt-network/poktroll/app/params" + appgateservercmd "github.com/pokt-network/poktroll/pkg/appgateserver/cmd" ) // NewRootCmd creates a new root command for a Cosmos SDK application @@ -148,6 +149,11 @@ func initRootCmd( txCommand(), keys.Commands(app.DefaultNodeHome), ) + + // add the appgate server command + rootCmd.AddCommand( + appgateservercmd.AppGateServerCmd(), + ) } // queryCommand returns the sub-command to send queries to the app diff --git a/docs/static/openapi.yml b/docs/static/openapi.yml index f55a851f2..d82f777d7 100644 --- a/docs/static/openapi.yml +++ b/docs/static/openapi.yml @@ -46480,7 +46480,7 @@ paths: service: title: >- The Service for which the application is - configured for + configured type: object properties: id: @@ -46660,9 +46660,7 @@ paths: type: object properties: service: - title: >- - The Service for which the application is configured - for + title: The Service for which the application is configured type: object properties: id: @@ -47178,7 +47176,7 @@ paths: service: title: >- The Service for which the application is - configured for + configured type: object properties: id: @@ -47245,7 +47243,7 @@ paths: service: title: >- The Service for which the supplier is - configured for + configured type: object properties: id: @@ -76789,7 +76787,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -76873,7 +76871,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -76967,7 +76965,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -77018,7 +77016,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -77286,7 +77284,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -77350,7 +77348,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured for + title: The Service for which the supplier is configured type: object properties: id: @@ -77540,7 +77538,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured for + title: The Service for which the application is configured type: object properties: id: @@ -77602,7 +77600,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured for + title: The Service for which the supplier is configured type: object properties: id: @@ -77816,7 +77814,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured for + title: The Service for which the supplier is configured type: object properties: id: @@ -77945,7 +77943,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured for + title: The Service for which the supplier is configured type: object properties: id: diff --git a/go.mod b/go.mod index da331e2f6..adbedcf85 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/athanorlabs/go-dleq v0.1.0 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 + github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -30,6 +31,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -73,7 +75,6 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect - github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -269,7 +270,6 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go new file mode 100644 index 000000000..943d2cf4f --- /dev/null +++ b/pkg/appgateserver/cmd/cmd.go @@ -0,0 +1,201 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "net/url" + "os" + "os/signal" + + "cosmossdk.io/depinject" + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" + ringtypes "github.com/athanorlabs/go-dleq/types" + cosmosclient "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/client/tx" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + "github.com/spf13/cobra" + + "github.com/pokt-network/poktroll/pkg/appgateserver" + blockclient "github.com/pokt-network/poktroll/pkg/client/block" + eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" +) + +var ( + signingKeyName string + listeningEndpoint string + cometWebsocketUrl string + selfSigning bool +) + +func AppGateServerCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "appgate-server", + Short: "Starts the AppGate server", + Long: `Starts the AppGate server that will listen for incoming relays requests and will handle +the interaction with the chain, sessions and suppliers in order to receive the correct +response for the request. + +If the server is started with the --self-signing flag, it will behave as an application +and sign any incoming requests with the private key associated with the --signing-key-name +flag provided. If however, this flag is not provided, the server will behave as a +gateway and will sign relays on behalf of any application sending it relays provided +that the address assoicated with the --signing-key-name flag has been delegated to by the +gateway, this is so that it can sign relays using the ring of the application. + +If an application doesn't provide the --self-signing flag, it will be able to send relays +to the AppGate server and it will still function as an application, however each request +will have to contain the "?senderAddress=[address]" query parameter, where [address] is +the address of the application that is sending the request. This is so that the server +can generate the correct ring for the application and sign the request.`, + Args: cobra.NoArgs, + RunE: runAppGateServer, + } + + cmd.Flags().StringVar(&signingKeyName, "signing-key-name", "", "The name of the key that will be used to sign relays") + cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:8080", "The host and port that the server will listen on") + cmd.Flags().BoolVar(&selfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") + cmd.Flags().StringVar(&cometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the tendermint websocket endpoint to interact with the chain") + + cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") + cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "tcp://: to tendermint rpc interface for this chain") + + return cmd +} + +func runAppGateServer(cmd *cobra.Command, _ []string) error { + // Validate the flags. + if signingKeyName == "" { + return fmt.Errorf("signing key name cannot be empty") + } + + // Create a context that is cancelled when the command is interrupted + ctx, cancelCtx := context.WithCancel(cmd.Context()) + + // Retrieve the client context for the chain interactions. + clientCtx := cosmosclient.GetClientContextFromCmd(cmd) + clientFactory, err := tx.NewFactoryCLI(clientCtx, cmd.Flags()) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to create tx factory: %w", err) + } + + // Get the signing key from the keyring. + key, err := clientFactory.Keybase().Key(signingKeyName) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to get key: %w", err) + } + + // Get the signing key address if the server is self-signing. + signingKeyAddr, err := key.GetAddress() + if err != nil { + cancelCtx() + return fmt.Errorf("failed to get address: %w", err) + } + + log.Printf("INFO: Signing key address: %s", signingKeyAddr.String()) + + // Obtain the private key's scalar point on the secp256k1 curve, to sign + // relay requests with the appropriate ring. + signingKey, err := recordLocalToScalar(key.GetLocal()) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to get signing key: %w", err) + } + + // Parse the listening endpoint. + listeningUrl, err := url.Parse(listeningEndpoint) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to parse listening endpoint: %w", err) + } + + // Obtain the tendermint websocket endpoint from the client context. + cometWSUrl, err := url.Parse(clientCtx.NodeURI + "/websocket") + if err != nil { + cancelCtx() + return fmt.Errorf("failed to parse block query URL: %w", err) + } + cometWSUrl.Scheme = "ws" + // If the comet websocket URL is not provided, use the one from the client context. + if cometWebsocketUrl == "" { + cometWebsocketUrl = cometWSUrl.String() + } + + log.Printf("INFO: Creating block client, using websocket URL: %s...", cometWebsocketUrl) + + // Create the block client with its dependency on the events client. + eventsQueryClient := eventsquery.NewEventsQueryClient(cometWebsocketUrl) + deps := depinject.Supply(eventsQueryClient) + blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to create block client: %w", err) + } + + log.Println("INFO: Creating AppGate server...") + + // Create the AppGate server. + signingKeyAddrStr := "" + if selfSigning { + signingKeyAddrStr = signingKeyAddr.String() + } + appGateServer := appgateserver.NewAppGateServer( + clientCtx, + signingKey, + signingKeyAddrStr, + listeningUrl, + blockClient, + ) + + // Handle interrupts in a goroutine. + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt) + // Block until we receive an interrupt or kill signal (OS-agnostic) + <-sigCh + log.Println("INFO: Interrupt signal received, shutting down...") + // Signal goroutines to stop + cancelCtx() + }() + + log.Printf("INFO: Starting AppGate server, listening on %s...", listeningUrl.String()) + + // Start the AppGate server. + if err := appGateServer.Start(ctx); err != nil && errors.Is(err, http.ErrServerClosed) { + cancelCtx() + return fmt.Errorf("failed to start app gate server: %w", err) + } else if errors.Is(err, http.ErrServerClosed) { + cancelCtx() + log.Println("INFO: AppGate server stopped") + } + + return nil +} + +// recordLocalToScalar converts the private key obtained from a +// key record to a scalar point on the secp256k1 curve +func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { + if local == nil { + return nil, fmt.Errorf("cannot extract private key from key record: nil") + } + priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) + if !ok { + return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) + } + if _, ok := priv.(*secp256k1.PrivKey); !ok { + return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) + } + crv := ring_secp256k1.NewCurve() + privKey, err := crv.DecodeToScalar(priv.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to decode private key: %w", err) + } + return privKey, nil +} diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index 2661945ce..a9c22659c 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -7,9 +7,12 @@ import ( ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" ring "github.com/noot/ring-go" + "github.com/pokt-network/poktroll/pkg/signer" apptypes "github.com/pokt-network/poktroll/x/application/types" ) @@ -103,8 +106,11 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str if err != nil { return nil, fmt.Errorf("unable to get account for address: %s [%w]", addr, err) } - acc := new(accounttypes.BaseAccount) - if err := acc.Unmarshal(pubKeyRes.Account.Value); err != nil { + var acc accounttypes.AccountI + reg := codectypes.NewInterfaceRegistry() + accounttypes.RegisterInterfaces(reg) + cdc := codec.NewProtoCodec(reg) + if err := cdc.UnpackAny(pubKeyRes.Account, &acc); err != nil { return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) } key := acc.GetPubKey() diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index 0c995a418..c1d583704 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -78,11 +78,11 @@ type appGateServer struct { supplierAccountCache map[string]cryptotypes.PubKey } -func NewAppServer( +func NewAppGateServer( clientCtx sdkclient.Context, signKey ringtypes.Scalar, appAddress string, - applicationEndpoint *url.URL, + listeningEndpoint *url.URL, blockClient blocktypes.BlockClient, ) *appGateServer { sessionQuerier := sessiontypes.NewQueryClient(clientCtx) @@ -96,10 +96,11 @@ func NewAppServer( appAddress: appAddress, clientCtx: clientCtx, sessionQuerier: sessionQuerier, + currentSessions: make(map[string]*sessiontypes.Session), accountQuerier: accountQuerier, applicationQuerier: applicationQuerier, blockClient: blockClient, - server: &http.Server{Addr: applicationEndpoint.Host}, + server: &http.Server{Addr: listeningEndpoint.Host}, supplierAccountCache: make(map[string]cryptotypes.PubKey), } } @@ -113,6 +114,9 @@ func (app *appGateServer) Start(ctx context.Context) error { app.server.Shutdown(ctx) }() + // Set the HTTP handler. + app.server.Handler = app + // Start the HTTP server. return app.server.ListenAndServe() } @@ -149,7 +153,7 @@ func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Re ) log.Print("ERROR: no application address provided") return - } else if appAddress != "" && appAddress != app.appAddress { + } else if appAddress != "" && app.appAddress != "" && appAddress != app.appAddress { app.replyWithError( writer, sdkerrors.Wrapf( diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index 18526508d..8a437f399 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -79,8 +79,7 @@ func NewBlockClient( ) (client.BlockClient, error) { // Initialize block client bClient := &blockClient{endpointURL: cometWebsocketURL} - bClient.latestBlockObsvbls, bClient.latestBlockObsvblsReplayPublishCh = - channel.NewReplayObservable[client.BlocksObservable](ctx, latestBlockObsvblsReplayBufferSize) + bClient.latestBlockObsvbls, bClient.latestBlockObsvblsReplayPublishCh = channel.NewReplayObservable[client.BlocksObservable](ctx, latestBlockObsvblsReplayBufferSize) // Inject dependencies if err := depinject.Inject(deps, &bClient.eventsClient); err != nil { @@ -138,7 +137,9 @@ func (bClient *blockClient) goPublishBlocks(ctx context.Context) { // If we get here, the retry limit was reached and the retry loop exited. // Since this function runs in a goroutine, we can't return the error to the // caller. Instead, we panic. - panic(fmt.Errorf("BlockClient.goPublishBlocks shold never reach this spot: %w", publishErr)) + if publishErr != nil { + panic(fmt.Errorf("BlockClient.goPublishBlocks should never reach this spot: %w", publishErr)) + } } // retryPublishBlocksFactory returns a function which is intended to be passed to From c22dc9c94b8ec3b71ba1a1380daddcbe60ca24c7 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 20:01:45 +0000 Subject: [PATCH 11/32] chore: add debug lines --- pkg/appgateserver/cmd/cmd.go | 4 ++-- pkg/appgateserver/jsonrpc.go | 11 +++++++++++ pkg/appgateserver/rings.go | 12 ++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 943d2cf4f..7048c2c9d 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -58,7 +58,7 @@ can generate the correct ring for the application and sign the request.`, } cmd.Flags().StringVar(&signingKeyName, "signing-key-name", "", "The name of the key that will be used to sign relays") - cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:8080", "The host and port that the server will listen on") + cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the server will listen on") cmd.Flags().BoolVar(&selfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") cmd.Flags().StringVar(&cometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the tendermint websocket endpoint to interact with the chain") @@ -168,7 +168,7 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { log.Printf("INFO: Starting AppGate server, listening on %s...", listeningUrl.String()) // Start the AppGate server. - if err := appGateServer.Start(ctx); err != nil && errors.Is(err, http.ErrServerClosed) { + if err := appGateServer.Start(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { cancelCtx() return fmt.Errorf("failed to start app gate server: %w", err) } else if errors.Is(err, http.ErrServerClosed) { diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 548ddb559..e815051d8 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "log" "net/http" "github.com/cometbft/cometbft/crypto" @@ -29,16 +30,20 @@ func (app *appGateServer) handleJSONRPCRelay( relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) + log.Printf("DEBUG: Getting current session for app [%s] and service [%s]...", appAddress, serviceId) session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { return err } + log.Printf("DEBUG: Current session ID: %s", session.SessionId) // Get a supplier URL and address for the given service and session. + log.Printf("DEBUG: Getting relayer URL for app [%s] and service [%s]...", appAddress, serviceId) supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, sharedtypes.RPCType_JSON_RPC, session) if err != nil { return err } + log.Printf("DEBUG: Relayer URL: %s", supplierUrl) // Create the relay request. relayRequest := &types.RelayRequest{ @@ -50,12 +55,14 @@ func (app *appGateServer) handleJSONRPCRelay( } // Get the application's signer. + log.Printf("DEBUG: Getting signer for app [%s]...", appAddress) signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { return err } // Hash and sign the request's signable bytes. + log.Printf("DEBUG: Signing relay request...") signableBz, err := relayRequest.GetSignableBytes() if err != nil { return err @@ -85,12 +92,14 @@ func (app *appGateServer) handleJSONRPCRelay( } // Perform the HTTP request to the relayer. + log.Printf("DEBUG: Sending relay request to relayer at [%s]...", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err } // Read the response body bytes. + log.Printf("DEBUG: Received relay response from relayer at [%s]...", supplierUrl) relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { return err @@ -106,6 +115,7 @@ func (app *appGateServer) handleJSONRPCRelay( // the getRelayerUrl function since this is the address we are expecting to sign the response. // TODO_TECHDEBT: if the RelayResponse is an internal error response, we should not verify the signature // as in some relayer early failures, it may not be signed by the supplier. + log.Printf("DEBUG: Verifying relay response signature...") if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { return err } @@ -117,6 +127,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Reply with the RelayResponse payload. + log.Printf("DEBUG: Sending relay response payload to app [%s]...", appAddress) if _, err := writer.Write(relayRequestBz); err != nil { return err } diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index a9c22659c..48e3bd987 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -4,6 +4,7 @@ package appgateserver import ( "context" "fmt" + "log" ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" @@ -68,6 +69,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( appAddress string, ) ([]ringtypes.Point, error) { // get the application's on chain state + log.Printf("DEBUG: Getting application for address: %s", appAddress) req := &apptypes.QueryGetApplicationRequest{Address: appAddress} res, err := app.applicationQuerier.Application(ctx, req) if err != nil { @@ -80,6 +82,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( copy(ringAddresses[1:], res.Application.DelegateeGatewayAddresses) // copy the gateway addresses // get the points on the secp256k1 curve for the addresses + log.Printf("DEBUG: Getting ring points for addresses: %v", ringAddresses) points, err := app.addressesToPoints(ctx, ringAddresses) if err != nil { return nil, err @@ -91,6 +94,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( app.ringCache[appAddress] = points // return the public key points on the secp256k1 curve + log.Print("DEBUG: Ring points: ", points) return points, nil } @@ -100,7 +104,8 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { curve := ring_secp256k1.NewCurve() points := make([]ringtypes.Point, len(addresses)) - for i, addr := range addresses { + for _, addr := range addresses { + log.Printf("DEBUG: Getting account for address: %s", addr) pubKeyReq := &accounttypes.QueryAccountRequest{Address: addr} pubKeyRes, err := app.accountQuerier.Account(ctx, pubKeyReq) if err != nil { @@ -110,6 +115,7 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str reg := codectypes.NewInterfaceRegistry() accounttypes.RegisterInterfaces(reg) cdc := codec.NewProtoCodec(reg) + log.Printf("DEBUG: Unpacking account for address: %s", addr) if err := cdc.UnpackAny(pubKeyRes.Account, &acc); err != nil { return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) } @@ -117,11 +123,13 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str if _, ok := key.(*secp256k1.PubKey); !ok { return nil, fmt.Errorf("public key is not a secp256k1 key: got %T", key) } + log.Printf("DEBUG: Decoding public key for address: %s", addr) point, err := curve.DecodeToPoint(key.Bytes()) if err != nil { return nil, err } - points[i] = point + log.Printf("DEBUG: Adding point to ring: %v", point) + points = append(points, point) } return points, nil } From ec57a8e07e8ac5a2b0d5e09e3d9787fa80b8fa15 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 20:39:54 +0000 Subject: [PATCH 12/32] chore: debugging --- pkg/appgateserver/jsonrpc.go | 12 +++++------- pkg/appgateserver/rings.go | 27 ++++++++++++++++----------- pkg/relayer/proxy/relay_signer.go | 4 +--- pkg/relayer/proxy/rings.go | 25 +++++++++++++++++-------- pkg/signer/interface.go | 2 +- pkg/signer/ring_signer.go | 9 +++++++-- pkg/signer/simple_signer.go | 2 +- 7 files changed, 48 insertions(+), 33 deletions(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index e815051d8..14b0349b0 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -55,7 +55,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Get the application's signer. - log.Printf("DEBUG: Getting signer for app [%s]...", appAddress) + log.Printf("DEBUG: Getting signer for app: %s...", appAddress) signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { return err @@ -68,9 +68,7 @@ func (app *appGateServer) handleJSONRPCRelay( return err } hash := crypto.Sha256(signableBz) - var hash32 [32]byte - copy(hash32[:], hash) - signature, err := signer.Sign(hash32) + signature, err := signer.Sign(hash) if err != nil { return err } @@ -92,14 +90,14 @@ func (app *appGateServer) handleJSONRPCRelay( } // Perform the HTTP request to the relayer. - log.Printf("DEBUG: Sending relay request to relayer at [%s]...", supplierUrl) + log.Printf("DEBUG: Sending relay request to relayer at: %s...", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err } // Read the response body bytes. - log.Printf("DEBUG: Received relay response from relayer at [%s]...", supplierUrl) + log.Printf("DEBUG: Received relay response from relayer at: %s...", supplierUrl) relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { return err @@ -127,7 +125,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Reply with the RelayResponse payload. - log.Printf("DEBUG: Sending relay response payload to app [%s]...", appAddress) + log.Printf("DEBUG: Sending relay response payload to app: %s...", appAddress) if _, err := writer.Write(relayRequestBz); err != nil { return err } diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index 48e3bd987..486979c09 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -33,9 +33,11 @@ func (app *appGateServer) getRingSingerForAppAddress(ctx context.Context, appAdd points, ok := app.ringCache[appAddress] if !ok { // if the ring is not in the cache, get it from the application module + log.Printf("DEBUG: No ring cached for address: %s", appAddress) ring, err = app.getRingForAppAddress(ctx, appAddress) } else { // if the ring is in the cache, create it from the points + log.Printf("DEBUG: Ring cached for address: %s", appAddress) ring, err = newRingFromPoints(points) } if err != nil { @@ -68,6 +70,9 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, ) ([]ringtypes.Point, error) { + app.ringCacheMutex.RLock() + defer app.ringCacheMutex.RUnlock() + // get the application's on chain state log.Printf("DEBUG: Getting application for address: %s", appAddress) req := &apptypes.QueryGetApplicationRequest{Address: appAddress} @@ -77,24 +82,24 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( } // create a slice of addresses for the ring - ringAddresses := make([]string, len(res.Application.DelegateeGatewayAddresses)+1) // +1 for app address - ringAddresses[0] = appAddress // app address is index 0 - copy(ringAddresses[1:], res.Application.DelegateeGatewayAddresses) // copy the gateway addresses + ringAddresses := make([]string, 0) + ringAddresses = append(ringAddresses, appAddress) // app address is index 0 + ringAddresses = append(ringAddresses, appAddress) // add app address twice to make the ring size of mininmum 2 + if len(res.Application.DelegateeGatewayAddresses) > 0 { + ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) // delegatee addresses are index 1+ + } // get the points on the secp256k1 curve for the addresses - log.Printf("DEBUG: Getting ring points for addresses: %v", ringAddresses) + log.Printf("DEBUG: Fetching public keys for ring: %v", ringAddresses) points, err := app.addressesToPoints(ctx, ringAddresses) if err != nil { return nil, err } // update the cache overwriting the previous value - app.ringCacheMutex.Lock() - defer app.ringCacheMutex.Unlock() app.ringCache[appAddress] = points // return the public key points on the secp256k1 curve - log.Print("DEBUG: Ring points: ", points) return points, nil } @@ -104,7 +109,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { curve := ring_secp256k1.NewCurve() points := make([]ringtypes.Point, len(addresses)) - for _, addr := range addresses { + for i, addr := range addresses { log.Printf("DEBUG: Getting account for address: %s", addr) pubKeyReq := &accounttypes.QueryAccountRequest{Address: addr} pubKeyRes, err := app.accountQuerier.Account(ctx, pubKeyReq) @@ -115,7 +120,7 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str reg := codectypes.NewInterfaceRegistry() accounttypes.RegisterInterfaces(reg) cdc := codec.NewProtoCodec(reg) - log.Printf("DEBUG: Unpacking account for address: %s", addr) + log.Printf("DEBUG: Unpacking account for address") if err := cdc.UnpackAny(pubKeyRes.Account, &acc); err != nil { return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) } @@ -123,13 +128,13 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str if _, ok := key.(*secp256k1.PubKey); !ok { return nil, fmt.Errorf("public key is not a secp256k1 key: got %T", key) } - log.Printf("DEBUG: Decoding public key for address: %s", addr) + log.Printf("DEBUG: Decoding public key for address") point, err := curve.DecodeToPoint(key.Bytes()) if err != nil { return nil, err } log.Printf("DEBUG: Adding point to ring: %v", point) - points = append(points, point) + points[i] = point } return points, nil } diff --git a/pkg/relayer/proxy/relay_signer.go b/pkg/relayer/proxy/relay_signer.go index abd0bc9ec..aba429fc2 100644 --- a/pkg/relayer/proxy/relay_signer.go +++ b/pkg/relayer/proxy/relay_signer.go @@ -23,11 +23,9 @@ func (rp *relayerProxy) SignRelayResponse(relayResponse *types.RelayResponse) er return sdkerrors.Wrapf(ErrInvalidRelayResponse, "error getting signable bytes: %v", err) } hash := crypto.Sha256(signableBz) - var hash32 [32]byte - copy(hash32[:], hash) // sign the relay response - sig, err := signer.Sign(hash32) + sig, err := signer.Sign(hash) if err != nil { return sdkerrors.Wrapf(ErrInvalidRelayResponse, "error signing relay response: %v", err) } diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index 2474fac9f..0df1183f5 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -19,15 +19,20 @@ import ( func (rp *relayerProxy) getRingForAppAddress(ctx context.Context, appAddress string) (*ring.Ring, error) { // lock the cache for reading rp.ringCacheMutex.RLock() - defer rp.ringCacheMutex.RUnlock() // check if the ring is in the cache points, ok := rp.ringCache[appAddress] + rp.ringCacheMutex.RUnlock() // unlock the cache incase not found in cache + var err error if !ok { // if the ring is not in the cache, get it from the application module - return rp.getRingForAppAddress(ctx, appAddress) + points, err = rp.getDelegatedPubKeysForAddress(ctx, appAddress) } - // if the ring is in the cache, create it from the points + if err != nil { + return nil, err + } + + // create the ring from the points return newRingFromPoints(points) } @@ -42,6 +47,9 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, ) ([]ringtypes.Point, error) { + rp.ringCacheMutex.RLock() + defer rp.ringCacheMutex.RUnlock() + // get the application's on chain state req := &apptypes.QueryGetApplicationRequest{Address: appAddress} res, err := rp.applicationQuerier.Application(ctx, req) @@ -50,9 +58,12 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( } // create a slice of addresses for the ring - ringAddresses := make([]string, len(res.Application.DelegateeGatewayAddresses)+1) // +1 for app address - ringAddresses[0] = appAddress // app address is index 0 - copy(ringAddresses[1:], res.Application.DelegateeGatewayAddresses) // copy the gateway addresses + ringAddresses := make([]string, 0) + ringAddresses = append(ringAddresses, appAddress) // app address is index 0 + ringAddresses = append(ringAddresses, appAddress) // add app address twice to make the ring size of mininmum 2 + if len(res.Application.DelegateeGatewayAddresses) > 0 { + ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) // delegatee addresses are index 1+ + } // get the points on the secp256k1 curve for the addresses points, err := rp.addressesToPoints(ctx, ringAddresses) @@ -61,8 +72,6 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( } // update the cache overwriting the previous value - rp.ringCacheMutex.Lock() - defer rp.ringCacheMutex.Unlock() rp.ringCache[appAddress] = points // return the public key points on the secp256k1 curve diff --git a/pkg/signer/interface.go b/pkg/signer/interface.go index 63b40b86d..5d91ae42a 100644 --- a/pkg/signer/interface.go +++ b/pkg/signer/interface.go @@ -5,5 +5,5 @@ package signer // The Signer interface expects a 32 byte message (sha256 hash) and returns a // byte slice containing the signature or any error that occurred during signing. type Signer interface { - Sign(msg [32]byte) (signature []byte, err error) + Sign(msg []byte) (signature []byte, err error) } diff --git a/pkg/signer/ring_signer.go b/pkg/signer/ring_signer.go index bb6cf549c..de401c25f 100644 --- a/pkg/signer/ring_signer.go +++ b/pkg/signer/ring_signer.go @@ -24,8 +24,13 @@ func NewRingSigner(ring *ring.Ring, privKey ringtypes.Scalar) *RingSigner { // Sign uses the ring and private key to sign the message provided and returns the // serialised ring signature that can be deserialised and verified by the verifier -func (r *RingSigner) Sign(msg [32]byte) ([]byte, error) { - ringSig, err := r.ring.Sign(msg, r.privKey) +func (r *RingSigner) Sign(msg []byte) ([]byte, error) { + if len(msg) != 32 { + return nil, fmt.Errorf("message must be 32 bytes long, got %d", len(msg)) + } + var msg32 [32]byte + copy(msg32[:], msg) + ringSig, err := r.ring.Sign(msg32, r.privKey) if err != nil { return nil, fmt.Errorf("failed to sign message [%v]: %w", msg, err) } diff --git a/pkg/signer/simple_signer.go b/pkg/signer/simple_signer.go index 432c1c8a3..208d56f67 100644 --- a/pkg/signer/simple_signer.go +++ b/pkg/signer/simple_signer.go @@ -17,7 +17,7 @@ func NewSimpleSigner(keyring keyring.Keyring, keyName string) *SimpleSigner { } // Sign signs the given message using the SimpleSigner's keyring and keyName -func (s *SimpleSigner) Sign(msg [32]byte) (signature []byte, err error) { +func (s *SimpleSigner) Sign(msg []byte) (signature []byte, err error) { sig, _, err := s.keyring.Sign(s.keyName, msg[:]) return sig, err } From 781ee130a6371200bf01c6ee64934a5388282127 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 20:41:41 +0000 Subject: [PATCH 13/32] chore: go.mod --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index da331e2f6..adbedcf85 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/athanorlabs/go-dleq v0.1.0 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 + github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -30,6 +31,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -73,7 +75,6 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect - github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -269,7 +270,6 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect From 078be2925d6aaec543a0d94eb76da9e248c2bb36 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 21:21:07 +0000 Subject: [PATCH 14/32] chore: close websocket connections --- pkg/appgateserver/cmd/cmd.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 7048c2c9d..923d00dcf 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -158,9 +158,15 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { go func() { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, os.Interrupt) + // Block until we receive an interrupt or kill signal (OS-agnostic) <-sigCh log.Println("INFO: Interrupt signal received, shutting down...") + + // close websocket connections + blockClient.Close() + eventsQueryClient.Close() + // Signal goroutines to stop cancelCtx() }() From 12d97d69f969c7ae4f8541f0f99b3d9890947c84 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 9 Nov 2023 22:17:15 +0000 Subject: [PATCH 15/32] chore: add ws todo --- pkg/appgateserver/cmd/cmd.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 923d00dcf..d4d7487fc 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -164,6 +164,8 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { log.Println("INFO: Interrupt signal received, shutting down...") // close websocket connections + // TODO(@h5law): Figure out why websockets arent closing properly + // in the sequencer logs blockClient.Close() eventsQueryClient.Close() From 3f2cb90683d43c6a315d67d8e21b8528ded9de1c Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 10 Nov 2023 14:25:33 +0100 Subject: [PATCH 16/32] chore: Use depinject for AppGateServer --- pkg/appgateserver/cmd/cmd.go | 100 +++--------------- pkg/appgateserver/endpoint_selector.go | 2 +- pkg/appgateserver/errors.go | 10 +- pkg/appgateserver/jsonrpc.go | 5 +- pkg/appgateserver/options.go | 17 +++ pkg/appgateserver/relay_verifier.go | 2 +- pkg/appgateserver/server.go | 138 +++++++++++++++++-------- x/service/types/relay.go | 18 +++- 8 files changed, 157 insertions(+), 135 deletions(-) create mode 100644 pkg/appgateserver/options.go diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index d4d7487fc..af3ba0a16 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -11,14 +11,8 @@ import ( "os/signal" "cosmossdk.io/depinject" - ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" - ringtypes "github.com/athanorlabs/go-dleq/types" cosmosclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" - "github.com/cosmos/cosmos-sdk/client/tx" - "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/spf13/cobra" "github.com/pokt-network/poktroll/pkg/appgateserver" @@ -30,7 +24,6 @@ var ( signingKeyName string listeningEndpoint string cometWebsocketUrl string - selfSigning bool ) func AppGateServerCmd() *cobra.Command { @@ -41,14 +34,14 @@ func AppGateServerCmd() *cobra.Command { the interaction with the chain, sessions and suppliers in order to receive the correct response for the request. -If the server is started with the --self-signing flag, it will behave as an application -and sign any incoming requests with the private key associated with the --signing-key-name -flag provided. If however, this flag is not provided, the server will behave as a -gateway and will sign relays on behalf of any application sending it relays provided -that the address assoicated with the --signing-key-name flag has been delegated to by the +If the server is started with a defined --signing-key-name flag, it will behave +as an application and sign any incoming requests with the private key associated with it. +If however, this flag is not provided, the server will behave as a gateway and will +sign relays on behalf of any application sending it relays provided +that the address associated with the --signing-key-name flag has been delegated to by the gateway, this is so that it can sign relays using the ring of the application. -If an application doesn't provide the --self-signing flag, it will be able to send relays +If an application doesn't provide the --signing-key-name flag, it will be able to send relays to the AppGate server and it will still function as an application, however each request will have to contain the "?senderAddress=[address]" query parameter, where [address] is the address of the application that is sending the request. This is so that the server @@ -59,7 +52,6 @@ can generate the correct ring for the application and sign the request.`, cmd.Flags().StringVar(&signingKeyName, "signing-key-name", "", "The name of the key that will be used to sign relays") cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the server will listen on") - cmd.Flags().BoolVar(&selfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") cmd.Flags().StringVar(&cometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the tendermint websocket endpoint to interact with the chain") cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") @@ -69,45 +61,11 @@ can generate the correct ring for the application and sign the request.`, } func runAppGateServer(cmd *cobra.Command, _ []string) error { - // Validate the flags. - if signingKeyName == "" { - return fmt.Errorf("signing key name cannot be empty") - } - // Create a context that is cancelled when the command is interrupted ctx, cancelCtx := context.WithCancel(cmd.Context()) // Retrieve the client context for the chain interactions. clientCtx := cosmosclient.GetClientContextFromCmd(cmd) - clientFactory, err := tx.NewFactoryCLI(clientCtx, cmd.Flags()) - if err != nil { - cancelCtx() - return fmt.Errorf("failed to create tx factory: %w", err) - } - - // Get the signing key from the keyring. - key, err := clientFactory.Keybase().Key(signingKeyName) - if err != nil { - cancelCtx() - return fmt.Errorf("failed to get key: %w", err) - } - - // Get the signing key address if the server is self-signing. - signingKeyAddr, err := key.GetAddress() - if err != nil { - cancelCtx() - return fmt.Errorf("failed to get address: %w", err) - } - - log.Printf("INFO: Signing key address: %s", signingKeyAddr.String()) - - // Obtain the private key's scalar point on the secp256k1 curve, to sign - // relay requests with the appropriate ring. - signingKey, err := recordLocalToScalar(key.GetLocal()) - if err != nil { - cancelCtx() - return fmt.Errorf("failed to get signing key: %w", err) - } // Parse the listening endpoint. listeningUrl, err := url.Parse(listeningEndpoint) @@ -142,18 +100,21 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { log.Println("INFO: Creating AppGate server...") // Create the AppGate server. - signingKeyAddrStr := "" - if selfSigning { - signingKeyAddrStr = signingKeyAddr.String() - } - appGateServer := appgateserver.NewAppGateServer( + appGateServerDeps := depinject.Supply( clientCtx, - signingKey, - signingKeyAddrStr, - listeningUrl, blockClient, ) + appGateServer, err := appgateserver.NewAppGateServer( + appGateServerDeps, + appgateserver.WithSigningKeyName(signingKeyName), + appgateserver.WithListeningUrl(listeningUrl), + ) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to create AppGate server: %w", err) + } + // Handle interrupts in a goroutine. go func() { sigCh := make(chan os.Signal, 1) @@ -163,12 +124,6 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { <-sigCh log.Println("INFO: Interrupt signal received, shutting down...") - // close websocket connections - // TODO(@h5law): Figure out why websockets arent closing properly - // in the sequencer logs - blockClient.Close() - eventsQueryClient.Close() - // Signal goroutines to stop cancelCtx() }() @@ -186,24 +141,3 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return nil } - -// recordLocalToScalar converts the private key obtained from a -// key record to a scalar point on the secp256k1 curve -func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { - if local == nil { - return nil, fmt.Errorf("cannot extract private key from key record: nil") - } - priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) - if !ok { - return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) - } - if _, ok := priv.(*secp256k1.PrivKey); !ok { - return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) - } - crv := ring_secp256k1.NewCurve() - privKey, err := crv.DecodeToScalar(priv.Bytes()) - if err != nil { - return nil, fmt.Errorf("failed to decode private key: %w", err) - } - return privKey, nil -} diff --git a/pkg/appgateserver/endpoint_selector.go b/pkg/appgateserver/endpoint_selector.go index 5e057741c..442d9fefd 100644 --- a/pkg/appgateserver/endpoint_selector.go +++ b/pkg/appgateserver/endpoint_selector.go @@ -41,5 +41,5 @@ func (app *appGateServer) getRelayerUrl( } // Return an error if no relayer endpoints were found. - return nil, "", ErrNoRelayEndpoints + return nil, "", ErrAppGateNoRelayEndpoints } diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index 7209f02c5..b02aa2ccc 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -3,8 +3,10 @@ package appgateserver import sdkerrors "cosmossdk.io/errors" var ( - codespace = "appclient" - ErrInvalidRelayResponseSignature = sdkerrors.Register(codespace, 1, "invalid relay response signature") - ErrNoRelayEndpoints = sdkerrors.Register(codespace, 2, "no relay endpoints found") - ErrInvalidRequestURL = sdkerrors.Register(codespace, 3, "invalid request URL") + codespace = "appgateserver" + ErrAppGateInvalidRelayResponseSignature = sdkerrors.Register(codespace, 1, "invalid relay response signature") + ErrAppGateNoRelayEndpoints = sdkerrors.Register(codespace, 2, "no relay endpoints found") + ErrAppGateInvalidRequestURL = sdkerrors.Register(codespace, 3, "invalid request URL") + ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 4, "missing application address") + ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 5, "missing app client listening endpoint") ) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 14b0349b0..a1cbb31f3 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -48,8 +48,8 @@ func (app *appGateServer) handleJSONRPCRelay( // Create the relay request. relayRequest := &types.RelayRequest{ Meta: &types.RelayRequestMetadata{ - // SessionHeader: session.Header, - Signature: nil, + SessionHeader: session.Header, + Signature: nil, }, Payload: relayRequestPayload, } @@ -67,6 +67,7 @@ func (app *appGateServer) handleJSONRPCRelay( if err != nil { return err } + hash := crypto.Sha256(signableBz) signature, err := signer.Sign(hash) if err != nil { diff --git a/pkg/appgateserver/options.go b/pkg/appgateserver/options.go new file mode 100644 index 000000000..313eeaa70 --- /dev/null +++ b/pkg/appgateserver/options.go @@ -0,0 +1,17 @@ +package appgateserver + +import "net/url" + +// WithSigningKeyName sets the signing key name for the app gate server. +func WithSigningKeyName(signingKeyName string) appGateServerOption { + return func(appGateServer *appGateServer) { + appGateServer.signingKeyName = signingKeyName + } +} + +// WithListeningUrl sets the listening URL for the app gate server. +func WithListeningUrl(listeningUrl *url.URL) appGateServerOption { + return func(appGateServer *appGateServer) { + appGateServer.listeningEndpoint = listeningUrl + } +} diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index d942a3704..76432ac64 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -34,7 +34,7 @@ func (app *appGateServer) verifyResponse( // Verify the relay response signature. if !pubKey.VerifySignature(hash, signature) { - return ErrInvalidRelayResponseSignature + return ErrAppGateInvalidRelayResponseSignature } return nil diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index c1d583704..9923e9ed5 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -2,15 +2,19 @@ package appgateserver import ( "context" + "fmt" "log" "net/http" "net/url" "strings" "sync" - sdkerrors "cosmossdk.io/errors" + "cosmossdk.io/depinject" + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" sdkclient "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" @@ -27,6 +31,9 @@ import ( // is running their own instance of the appGateServer or they are sending requests to a gateway running an // instance of the appGateServer, they will need to either include the application address in the request or not. type appGateServer struct { + // signingKeyName is the name of the key that will be used to sign relays + signingKeyName string + // signingKey is the scalar point on the appropriate curve corresponding to the // signer's private key, and is used to sign relay requests via a ring signature signingKey ringtypes.Scalar @@ -69,6 +76,9 @@ type appGateServer struct { // It is used to get the current block height to query for the current session. blockClient blocktypes.BlockClient + // listeningEndpoint is the endpoint that the appGateServer will listen on. + listeningEndpoint *url.URL + // server is the HTTP server that will be used capture application requests // so that they can be signed and relayed to the supplier. server *http.Server @@ -79,30 +89,55 @@ type appGateServer struct { } func NewAppGateServer( - clientCtx sdkclient.Context, - signKey ringtypes.Scalar, - appAddress string, - listeningEndpoint *url.URL, - blockClient blocktypes.BlockClient, -) *appGateServer { - sessionQuerier := sessiontypes.NewQueryClient(clientCtx) - accountQuerier := accounttypes.NewQueryClient(clientCtx) - applicationQuerier := apptypes.NewQueryClient(clientCtx) - - return &appGateServer{ - signingKey: signKey, + deps depinject.Config, + opts ...appGateServerOption, +) (*appGateServer, error) { + app := &appGateServer{ ringCacheMutex: &sync.RWMutex{}, ringCache: make(map[string][]ringtypes.Point), - appAddress: appAddress, - clientCtx: clientCtx, - sessionQuerier: sessionQuerier, currentSessions: make(map[string]*sessiontypes.Session), - accountQuerier: accountQuerier, - applicationQuerier: applicationQuerier, - blockClient: blockClient, - server: &http.Server{Addr: listeningEndpoint.Host}, supplierAccountCache: make(map[string]cryptotypes.PubKey), } + + if err := depinject.Inject( + deps, + &app.clientCtx, + &app.blockClient, + ); err != nil { + return nil, err + } + + for _, opt := range opts { + opt(app) + } + + if err := app.validateConfig(); err != nil { + return nil, err + } + + key, err := app.clientCtx.Keyring.Key(app.signingKeyName) + if err != nil { + return nil, err + } + + appAddress, err := key.GetAddress() + if err != nil { + return nil, err + } + app.appAddress = appAddress.String() + + signingKey, err := recordLocalToScalar(key.GetLocal()) + if err != nil { + return nil, err + } + app.signingKey = signingKey + + app.sessionQuerier = sessiontypes.NewQueryClient(app.clientCtx) + app.accountQuerier = accounttypes.NewQueryClient(app.clientCtx) + app.applicationQuerier = apptypes.NewQueryClient(app.clientCtx) + app.server = &http.Server{Addr: app.listeningEndpoint.Host} + + return app, nil } // Start starts the application server and blocks until the context is done @@ -143,31 +178,19 @@ func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Re // Extract the serviceId from the request path. path := request.URL.Path serviceId := strings.Split(path, "/")[1] - appAddress := request.URL.Query().Get("senderAddr") - - // ensure the app address is present - if appAddress == "" && app.appAddress == "" { - app.replyWithError( - writer, - sdkerrors.Wrapf(ErrInvalidRequestURL, "missing sender address query parameter: got %s", request.URL.String()), - ) - log.Print("ERROR: no application address provided") - return - } else if appAddress != "" && app.appAddress != "" && appAddress != app.appAddress { - app.replyWithError( - writer, - sdkerrors.Wrapf( - ErrInvalidRequestURL, - "sender address query parameter does not match the application address: got %s, want %s", - appAddress, app.appAddress, - ), - ) - log.Print("ERROR: application address does not match sender address query parameter") - return - } else if appAddress == "" { + var appAddress string + + if app.appAddress != "" { + appAddress = request.URL.Query().Get("senderAddr") + } else { appAddress = app.appAddress } + if appAddress == "" { + app.replyWithError(writer, ErrAppGateMissingAppAddress) + log.Print("ERROR: no application address provided") + } + // TODO_TECHDEBT: Currently, there is no information about the RPC type requested. It should // be extracted from the request and used to determine the RPC type to handle. handle*Relay() // calls should be wrapped into a switch statement to handle different types of relays. @@ -212,3 +235,34 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) return } } + +// validateConfig validates the appGateServer configuration. +func (app *appGateServer) validateConfig() error { + if app.listeningEndpoint == nil { + return ErrAppGateMissingListeningEndpoint + } + return nil +} + +// recordLocalToScalar converts the private key obtained from a +// key record to a scalar point on the secp256k1 curve +func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { + if local == nil { + return nil, fmt.Errorf("cannot extract private key from key record: nil") + } + priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) + if !ok { + return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) + } + if _, ok := priv.(*secp256k1.PrivKey); !ok { + return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) + } + crv := ring_secp256k1.NewCurve() + privKey, err := crv.DecodeToScalar(priv.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to decode private key: %w", err) + } + return privKey, nil +} + +type appGateServerOption func(*appGateServer) diff --git a/x/service/types/relay.go b/x/service/types/relay.go index 9c0c4809f..d8928bb34 100644 --- a/x/service/types/relay.go +++ b/x/service/types/relay.go @@ -5,9 +5,16 @@ package types // A value receiver is used to avoid overwriting any pre-existing signature func (req RelayRequest) GetSignableBytes() ([]byte, error) { // set signature to nil + sig := req.Meta.Signature req.Meta.Signature = nil + + bz, err := req.Marshal() + if err == nil { + req.Meta.Signature = sig + } + // return the marshaled message - return req.Marshal() + return bz, nil } // GetSignableBytes returns the signable bytes for the relay response @@ -15,7 +22,14 @@ func (req RelayRequest) GetSignableBytes() ([]byte, error) { // A value receiver is used to avoid overwriting any pre-existing signature func (res RelayResponse) GetSignableBytes() ([]byte, error) { // set signature to nil + sig := res.Meta.SupplierSignature res.Meta.SupplierSignature = nil + + bz, err := res.Marshal() + if err == nil { + res.Meta.SupplierSignature = sig + } + // return the marshaled message - return res.Marshal() + return bz, err } From a857e5fd63e0d211748ced27e7c97bbaef3c55ee Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 10 Nov 2023 14:32:29 +0100 Subject: [PATCH 17/32] fix: Get appAddress from url query when appAddress is empty --- pkg/appgateserver/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index 9923e9ed5..f5be1bbf8 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -180,7 +180,7 @@ func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Re serviceId := strings.Split(path, "/")[1] var appAddress string - if app.appAddress != "" { + if app.appAddress == "" { appAddress = request.URL.Query().Get("senderAddr") } else { appAddress = app.appAddress From bc2efb37d5fc1d46641298c7d749a2d7d950793a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:03:10 +0000 Subject: [PATCH 18/32] feat: address comments --- pkg/appgateserver/cmd/cmd.go | 72 ++++++++++++++++++++++++++++++++---- pkg/appgateserver/errors.go | 3 +- pkg/appgateserver/options.go | 10 +++-- pkg/appgateserver/server.go | 71 +++++++++-------------------------- x/service/types/relay.go | 16 +------- 5 files changed, 91 insertions(+), 81 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index af3ba0a16..0f67f274b 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -11,8 +11,13 @@ import ( "os/signal" "cosmossdk.io/depinject" + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" + ringtypes "github.com/athanorlabs/go-dleq/types" cosmosclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/spf13/cobra" "github.com/pokt-network/poktroll/pkg/appgateserver" @@ -22,6 +27,7 @@ import ( var ( signingKeyName string + selfSigning bool listeningEndpoint string cometWebsocketUrl string ) @@ -34,14 +40,15 @@ func AppGateServerCmd() *cobra.Command { the interaction with the chain, sessions and suppliers in order to receive the correct response for the request. -If the server is started with a defined --signing-key-name flag, it will behave -as an application and sign any incoming requests with the private key associated with it. -If however, this flag is not provided, the server will behave as a gateway and will -sign relays on behalf of any application sending it relays provided -that the address associated with the --signing-key-name flag has been delegated to by the -gateway, this is so that it can sign relays using the ring of the application. +If the server is started with a defined --self-signing flag, it will behave +as an application and sign any incoming requests with the private key associated with +the --signing-key-name flag. If however, this flag is not provided, the server will +behave as a gateway and will sign relays on behalf of any application sending it relays provided +that the address recieved in the query parameters of a request has been delegated to by the +gateway, this is so that it can sign relays using the ring of the application with the +key associated with the --signing-key-name flag. -If an application doesn't provide the --signing-key-name flag, it will be able to send relays +If an application doesn't provide the --self-signing flag, it will be able to send relays to the AppGate server and it will still function as an application, however each request will have to contain the "?senderAddress=[address]" query parameter, where [address] is the address of the application that is sending the request. This is so that the server @@ -53,6 +60,7 @@ can generate the correct ring for the application and sign the request.`, cmd.Flags().StringVar(&signingKeyName, "signing-key-name", "", "The name of the key that will be used to sign relays") cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the server will listen on") cmd.Flags().StringVar(&cometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the tendermint websocket endpoint to interact with the chain") + cmd.Flags().BoolVar(&selfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "tcp://: to tendermint rpc interface for this chain") @@ -99,6 +107,33 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { log.Println("INFO: Creating AppGate server...") + key, err := clientCtx.Keyring.Key(signingKeyName) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to get key from keyring: %w", err) + } + + appAddress, err := key.GetAddress() + if err != nil { + cancelCtx() + return fmt.Errorf("failed to get address from key: %w", err) + } + signingAddress := "" + if selfSigning { + signingAddress = appAddress.String() + } + + signingKey, err := recordLocalToScalar(key.GetLocal()) + if err != nil { + cancelCtx() + return fmt.Errorf("failed to convert private key to scalar: %w", err) + } + signingKey = signingKey + signingInfo := appgateserver.SigningInformation{ + SigningKey: signingKey, + AppAddress: signingAddress, + } + // Create the AppGate server. appGateServerDeps := depinject.Supply( clientCtx, @@ -107,7 +142,7 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { appGateServer, err := appgateserver.NewAppGateServer( appGateServerDeps, - appgateserver.WithSigningKeyName(signingKeyName), + appgateserver.WithSigningInformation(&signingInfo), appgateserver.WithListeningUrl(listeningUrl), ) if err != nil { @@ -141,3 +176,24 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return nil } + +// recordLocalToScalar converts the private key obtained from a +// key record to a scalar point on the secp256k1 curve +func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { + if local == nil { + return nil, fmt.Errorf("cannot extract private key from key record: nil") + } + priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) + if !ok { + return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) + } + if _, ok := priv.(*secp256k1.PrivKey); !ok { + return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) + } + crv := ring_secp256k1.NewCurve() + privKey, err := crv.DecodeToScalar(priv.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to decode private key: %w", err) + } + return privKey, nil +} diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index b02aa2ccc..2c8f281bd 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -8,5 +8,6 @@ var ( ErrAppGateNoRelayEndpoints = sdkerrors.Register(codespace, 2, "no relay endpoints found") ErrAppGateInvalidRequestURL = sdkerrors.Register(codespace, 3, "invalid request URL") ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 4, "missing application address") - ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 5, "missing app client listening endpoint") + ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 5, "missing app client signing information") + ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 6, "missing app client listening endpoint") ) diff --git a/pkg/appgateserver/options.go b/pkg/appgateserver/options.go index 313eeaa70..33f0c90a5 100644 --- a/pkg/appgateserver/options.go +++ b/pkg/appgateserver/options.go @@ -1,11 +1,13 @@ package appgateserver -import "net/url" +import ( + "net/url" +) -// WithSigningKeyName sets the signing key name for the app gate server. -func WithSigningKeyName(signingKeyName string) appGateServerOption { +// WithSigningInformation sets the signing key and app address for server. +func WithSigningInformation(signingInfo *SigningInformation) appGateServerOption { return func(appGateServer *appGateServer) { - appGateServer.signingKeyName = signingKeyName + appGateServer.signingInformation = signingInfo } } diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index f5be1bbf8..f01a876b7 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -2,7 +2,6 @@ package appgateserver import ( "context" - "fmt" "log" "net/http" "net/url" @@ -10,11 +9,8 @@ import ( "sync" "cosmossdk.io/depinject" - ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" sdkclient "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" @@ -24,6 +20,16 @@ import ( sessiontypes "github.com/pokt-network/poktroll/x/session/types" ) +type SigningInformation struct { + // SigningKey is the scalar point on the appropriate curve corresponding to the + // signer's private key, and is used to sign relay requests via a ring signature + SigningKey ringtypes.Scalar + + // AppAddress is the address of the application that the server is serving if + // it is nil then the application address must be included in each request + AppAddress string +} + // appGateServer is the server that listens for application requests and relays them to the supplier. // it is responsible for maintaining the current session for the application, signing the requests, // and verifying the response signatures. @@ -31,12 +37,8 @@ import ( // is running their own instance of the appGateServer or they are sending requests to a gateway running an // instance of the appGateServer, they will need to either include the application address in the request or not. type appGateServer struct { - // signingKeyName is the name of the key that will be used to sign relays - signingKeyName string - - // signingKey is the scalar point on the appropriate curve corresponding to the - // signer's private key, and is used to sign relay requests via a ring signature - signingKey ringtypes.Scalar + // signing information holds the signing key and application address for the server + signingInformation *SigningInformation // ringCache is a cache of the public keys used to create the ring for a given application // they are stored in a map of application address to a slice of points on the secp256k1 curve @@ -44,10 +46,6 @@ type appGateServer struct { ringCache map[string][]ringtypes.Point ringCacheMutex *sync.RWMutex - // appAddress is the address of the application that the server is serving if - // it is nil then the application address must be included in each request - appAddress string - // clientCtx is the client context for the application. // It is used to query for the application's account to unmarshal the supplier's account // and get the public key to verify the relay response signature. @@ -115,23 +113,6 @@ func NewAppGateServer( return nil, err } - key, err := app.clientCtx.Keyring.Key(app.signingKeyName) - if err != nil { - return nil, err - } - - appAddress, err := key.GetAddress() - if err != nil { - return nil, err - } - app.appAddress = appAddress.String() - - signingKey, err := recordLocalToScalar(key.GetLocal()) - if err != nil { - return nil, err - } - app.signingKey = signingKey - app.sessionQuerier = sessiontypes.NewQueryClient(app.clientCtx) app.accountQuerier = accounttypes.NewQueryClient(app.clientCtx) app.applicationQuerier = apptypes.NewQueryClient(app.clientCtx) @@ -180,10 +161,10 @@ func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Re serviceId := strings.Split(path, "/")[1] var appAddress string - if app.appAddress == "" { + if app.signingInformation.AppAddress == "" { appAddress = request.URL.Query().Get("senderAddr") } else { - appAddress = app.appAddress + appAddress = app.signingInformation.AppAddress } if appAddress == "" { @@ -238,31 +219,13 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) // validateConfig validates the appGateServer configuration. func (app *appGateServer) validateConfig() error { + if app.signingInformation == nil { + return ErrAppGateMissingSigningInformation + } if app.listeningEndpoint == nil { return ErrAppGateMissingListeningEndpoint } return nil } -// recordLocalToScalar converts the private key obtained from a -// key record to a scalar point on the secp256k1 curve -func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { - if local == nil { - return nil, fmt.Errorf("cannot extract private key from key record: nil") - } - priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) - if !ok { - return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) - } - if _, ok := priv.(*secp256k1.PrivKey); !ok { - return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) - } - crv := ring_secp256k1.NewCurve() - privKey, err := crv.DecodeToScalar(priv.Bytes()) - if err != nil { - return nil, fmt.Errorf("failed to decode private key: %w", err) - } - return privKey, nil -} - type appGateServerOption func(*appGateServer) diff --git a/x/service/types/relay.go b/x/service/types/relay.go index d8928bb34..c7b2a1894 100644 --- a/x/service/types/relay.go +++ b/x/service/types/relay.go @@ -5,16 +5,10 @@ package types // A value receiver is used to avoid overwriting any pre-existing signature func (req RelayRequest) GetSignableBytes() ([]byte, error) { // set signature to nil - sig := req.Meta.Signature req.Meta.Signature = nil - bz, err := req.Marshal() - if err == nil { - req.Meta.Signature = sig - } - // return the marshaled message - return bz, nil + return req.Marshal() } // GetSignableBytes returns the signable bytes for the relay response @@ -22,14 +16,8 @@ func (req RelayRequest) GetSignableBytes() ([]byte, error) { // A value receiver is used to avoid overwriting any pre-existing signature func (res RelayResponse) GetSignableBytes() ([]byte, error) { // set signature to nil - sig := res.Meta.SupplierSignature res.Meta.SupplierSignature = nil - bz, err := res.Marshal() - if err == nil { - res.Meta.SupplierSignature = sig - } - // return the marshaled message - return bz, err + return res.Marshal() } From 76a31c07390081c2e6db8240722f69eacb25c00e Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:10:52 +0000 Subject: [PATCH 19/32] chore: fix signing key field --- pkg/appgateserver/rings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index 486979c09..adabe0a09 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -45,7 +45,7 @@ func (app *appGateServer) getRingSingerForAppAddress(ctx context.Context, appAdd } // return the ring signer - return signer.NewRingSigner(ring, app.signingKey), nil + return signer.NewRingSigner(ring, app.signingInformation.SigningKey), nil } // getRingForAppAddress returns the RingSinger used to sign relays. It does so by fetching From 09c843ccae6873e136a2193bfe555bc98c4eb007 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:29:07 +0000 Subject: [PATCH 20/32] chore: defer cancelling ctx --- pkg/appgateserver/cmd/cmd.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 0f67f274b..9a50a65b4 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -71,6 +71,7 @@ can generate the correct ring for the application and sign the request.`, func runAppGateServer(cmd *cobra.Command, _ []string) error { // Create a context that is cancelled when the command is interrupted ctx, cancelCtx := context.WithCancel(cmd.Context()) + defer cancelCtx() // Retrieve the client context for the chain interactions. clientCtx := cosmosclient.GetClientContextFromCmd(cmd) @@ -78,14 +79,12 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { // Parse the listening endpoint. listeningUrl, err := url.Parse(listeningEndpoint) if err != nil { - cancelCtx() return fmt.Errorf("failed to parse listening endpoint: %w", err) } // Obtain the tendermint websocket endpoint from the client context. cometWSUrl, err := url.Parse(clientCtx.NodeURI + "/websocket") if err != nil { - cancelCtx() return fmt.Errorf("failed to parse block query URL: %w", err) } cometWSUrl.Scheme = "ws" @@ -101,7 +100,6 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { deps := depinject.Supply(eventsQueryClient) blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) if err != nil { - cancelCtx() return fmt.Errorf("failed to create block client: %w", err) } @@ -109,13 +107,11 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { key, err := clientCtx.Keyring.Key(signingKeyName) if err != nil { - cancelCtx() return fmt.Errorf("failed to get key from keyring: %w", err) } appAddress, err := key.GetAddress() if err != nil { - cancelCtx() return fmt.Errorf("failed to get address from key: %w", err) } signingAddress := "" @@ -125,10 +121,8 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { signingKey, err := recordLocalToScalar(key.GetLocal()) if err != nil { - cancelCtx() return fmt.Errorf("failed to convert private key to scalar: %w", err) } - signingKey = signingKey signingInfo := appgateserver.SigningInformation{ SigningKey: signingKey, AppAddress: signingAddress, @@ -146,7 +140,6 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { appgateserver.WithListeningUrl(listeningUrl), ) if err != nil { - cancelCtx() return fmt.Errorf("failed to create AppGate server: %w", err) } @@ -167,10 +160,8 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { // Start the AppGate server. if err := appGateServer.Start(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { - cancelCtx() return fmt.Errorf("failed to start app gate server: %w", err) } else if errors.Is(err, http.ErrServerClosed) { - cancelCtx() log.Println("INFO: AppGate server stopped") } From 151313e2de99cdbc24fd026cd8ed3b2c7b18c3f1 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:34:27 +0000 Subject: [PATCH 21/32] chore: cleanup log lines --- pkg/appgateserver/jsonrpc.go | 10 +--------- pkg/appgateserver/rings.go | 7 +------ 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index a1cbb31f3..2c8144ddc 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -30,7 +30,6 @@ func (app *appGateServer) handleJSONRPCRelay( relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) - log.Printf("DEBUG: Getting current session for app [%s] and service [%s]...", appAddress, serviceId) session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { return err @@ -38,12 +37,10 @@ func (app *appGateServer) handleJSONRPCRelay( log.Printf("DEBUG: Current session ID: %s", session.SessionId) // Get a supplier URL and address for the given service and session. - log.Printf("DEBUG: Getting relayer URL for app [%s] and service [%s]...", appAddress, serviceId) supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, sharedtypes.RPCType_JSON_RPC, session) if err != nil { return err } - log.Printf("DEBUG: Relayer URL: %s", supplierUrl) // Create the relay request. relayRequest := &types.RelayRequest{ @@ -55,14 +52,12 @@ func (app *appGateServer) handleJSONRPCRelay( } // Get the application's signer. - log.Printf("DEBUG: Getting signer for app: %s...", appAddress) signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { return err } // Hash and sign the request's signable bytes. - log.Printf("DEBUG: Signing relay request...") signableBz, err := relayRequest.GetSignableBytes() if err != nil { return err @@ -91,14 +86,13 @@ func (app *appGateServer) handleJSONRPCRelay( } // Perform the HTTP request to the relayer. - log.Printf("DEBUG: Sending relay request to relayer at: %s...", supplierUrl) + log.Printf("DEBUG: Sending relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err } // Read the response body bytes. - log.Printf("DEBUG: Received relay response from relayer at: %s...", supplierUrl) relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { return err @@ -114,7 +108,6 @@ func (app *appGateServer) handleJSONRPCRelay( // the getRelayerUrl function since this is the address we are expecting to sign the response. // TODO_TECHDEBT: if the RelayResponse is an internal error response, we should not verify the signature // as in some relayer early failures, it may not be signed by the supplier. - log.Printf("DEBUG: Verifying relay response signature...") if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { return err } @@ -126,7 +119,6 @@ func (app *appGateServer) handleJSONRPCRelay( } // Reply with the RelayResponse payload. - log.Printf("DEBUG: Sending relay response payload to app: %s...", appAddress) if _, err := writer.Write(relayRequestBz); err != nil { return err } diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index adabe0a09..0acf423e8 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -41,6 +41,7 @@ func (app *appGateServer) getRingSingerForAppAddress(ctx context.Context, appAdd ring, err = newRingFromPoints(points) } if err != nil { + log.Printf("ERROR: Unable to get ring for address: %s [%w]", appAddress, err) return nil, err } @@ -74,7 +75,6 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( defer app.ringCacheMutex.RUnlock() // get the application's on chain state - log.Printf("DEBUG: Getting application for address: %s", appAddress) req := &apptypes.QueryGetApplicationRequest{Address: appAddress} res, err := app.applicationQuerier.Application(ctx, req) if err != nil { @@ -90,7 +90,6 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( } // get the points on the secp256k1 curve for the addresses - log.Printf("DEBUG: Fetching public keys for ring: %v", ringAddresses) points, err := app.addressesToPoints(ctx, ringAddresses) if err != nil { return nil, err @@ -110,7 +109,6 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str curve := ring_secp256k1.NewCurve() points := make([]ringtypes.Point, len(addresses)) for i, addr := range addresses { - log.Printf("DEBUG: Getting account for address: %s", addr) pubKeyReq := &accounttypes.QueryAccountRequest{Address: addr} pubKeyRes, err := app.accountQuerier.Account(ctx, pubKeyReq) if err != nil { @@ -120,7 +118,6 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str reg := codectypes.NewInterfaceRegistry() accounttypes.RegisterInterfaces(reg) cdc := codec.NewProtoCodec(reg) - log.Printf("DEBUG: Unpacking account for address") if err := cdc.UnpackAny(pubKeyRes.Account, &acc); err != nil { return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) } @@ -128,12 +125,10 @@ func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []str if _, ok := key.(*secp256k1.PubKey); !ok { return nil, fmt.Errorf("public key is not a secp256k1 key: got %T", key) } - log.Printf("DEBUG: Decoding public key for address") point, err := curve.DecodeToPoint(key.Bytes()) if err != nil { return nil, err } - log.Printf("DEBUG: Adding point to ring: %v", point) points[i] = point } return points, nil From 2cd04e719ec2840ab76966326930215eca9f37ee Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:10:42 +0000 Subject: [PATCH 22/32] chore: address comments --- pkg/appgateserver/cmd/cmd.go | 83 +++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 9a50a65b4..bda778da7 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -26,50 +26,54 @@ import ( ) var ( - signingKeyName string - selfSigning bool - listeningEndpoint string - cometWebsocketUrl string + flagSigningKey string + flagSelfSigning bool + flagListeningEndpoint string + flagCometWebsocketUrl string ) func AppGateServerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "appgate-server", Short: "Starts the AppGate server", - Long: `Starts the AppGate server that will listen for incoming relays requests and will handle -the interaction with the chain, sessions and suppliers in order to receive the correct -response for the request. - -If the server is started with a defined --self-signing flag, it will behave -as an application and sign any incoming requests with the private key associated with -the --signing-key-name flag. If however, this flag is not provided, the server will -behave as a gateway and will sign relays on behalf of any application sending it relays provided -that the address recieved in the query parameters of a request has been delegated to by the -gateway, this is so that it can sign relays using the ring of the application with the -key associated with the --signing-key-name flag. - -If an application doesn't provide the --self-signing flag, it will be able to send relays -to the AppGate server and it will still function as an application, however each request -will have to contain the "?senderAddress=[address]" query parameter, where [address] is -the address of the application that is sending the request. This is so that the server -can generate the correct ring for the application and sign the request.`, + Long: `Starts the AppGate server that listens for incoming relay requests and handles +the necessary on-chain interactions (sessions, suppliers, etc) to receive the +respective relay response. + +-- App Mode (Flag)- - +If the server is started with a defined '--self-signing' flag, it will behave +as an Application. Any incoming requests will be signed by using the private +key and ring associated with the '--signing-key' flag. + +-- Gateway Mode (Flag)-- +If the '--self-signing' flag is not provided, the server will behave as a Gateway. +It will sign relays on behalf of any Application sending it relays, provided +that the address associated with '--signing-key' has been delegated to. This is +necessary for the application<->gateway ring signature to function. + +-- App Mode (HTTP) -- +If an application doesn't provide the '--self-signing' flag, it can still send +relays to the AppGate server and function as an Application, provided that: +1. Each request contains the '?senderAddress=[address]' query parameter +2. The key associated with the '--signing-key' flag belongs to the address + provided in the request, otherwise the ring signature will not be valid.`, Args: cobra.NoArgs, RunE: runAppGateServer, } - cmd.Flags().StringVar(&signingKeyName, "signing-key-name", "", "The name of the key that will be used to sign relays") - cmd.Flags().StringVar(&listeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the server will listen on") - cmd.Flags().StringVar(&cometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the tendermint websocket endpoint to interact with the chain") - cmd.Flags().BoolVar(&selfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") + cmd.Flags().StringVar(&flagSigningKey, "signing-key", "", "The name of the key that will be used to sign relays") + cmd.Flags().StringVar(&flagListeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the appgate server will listen on") + cmd.Flags().StringVar(&flagCometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the comet websocket endpoint to communicate with the pocket blockchain") + cmd.Flags().BoolVar(&flagSelfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "tcp://: to tendermint rpc interface for this chain") + cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "The URL of the comet tcp endpoint to communicate with the pocket blockchain") return cmd } func runAppGateServer(cmd *cobra.Command, _ []string) error { - // Create a context that is cancelled when the command is interrupted + // Create a context that is canceled when the command is interrupted ctx, cancelCtx := context.WithCancel(cmd.Context()) defer cancelCtx() @@ -77,49 +81,52 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { clientCtx := cosmosclient.GetClientContextFromCmd(cmd) // Parse the listening endpoint. - listeningUrl, err := url.Parse(listeningEndpoint) + listeningUrl, err := url.Parse(flagListeningEndpoint) if err != nil { return fmt.Errorf("failed to parse listening endpoint: %w", err) } - // Obtain the tendermint websocket endpoint from the client context. + // Obtain the comet websocket endpoint from the client context. cometWSUrl, err := url.Parse(clientCtx.NodeURI + "/websocket") if err != nil { return fmt.Errorf("failed to parse block query URL: %w", err) } cometWSUrl.Scheme = "ws" // If the comet websocket URL is not provided, use the one from the client context. - if cometWebsocketUrl == "" { - cometWebsocketUrl = cometWSUrl.String() + if flagCometWebsocketUrl == "" { + flagCometWebsocketUrl = cometWSUrl.String() } - log.Printf("INFO: Creating block client, using websocket URL: %s...", cometWebsocketUrl) + log.Printf("INFO: Creating block client, using comet websocket URL: %s...", flagCometWebsocketUrl) // Create the block client with its dependency on the events client. - eventsQueryClient := eventsquery.NewEventsQueryClient(cometWebsocketUrl) + eventsQueryClient := eventsquery.NewEventsQueryClient(flagCometWebsocketUrl) deps := depinject.Supply(eventsQueryClient) - blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) + blockClient, err := blockclient.NewBlockClient(ctx, deps, flagCometWebsocketUrl) if err != nil { return fmt.Errorf("failed to create block client: %w", err) } log.Println("INFO: Creating AppGate server...") - key, err := clientCtx.Keyring.Key(signingKeyName) + keyRecord, err := clientCtx.Keyring.Key(flagSigningKey) if err != nil { return fmt.Errorf("failed to get key from keyring: %w", err) } - appAddress, err := key.GetAddress() + appAddress, err := keyRecord.GetAddress() if err != nil { return fmt.Errorf("failed to get address from key: %w", err) } signingAddress := "" - if selfSigning { + if flagSelfSigning { signingAddress = appAddress.String() } - signingKey, err := recordLocalToScalar(key.GetLocal()) + // Convert the key record to a private key and return the scalar + // point on the secp256k1 curve that it corresponds to. + // If the key is not a secp256k1 key, this will return an error. + signingKey, err := recordLocalToScalar(keyRecord.GetLocal()) if err != nil { return fmt.Errorf("failed to convert private key to scalar: %w", err) } From 5daed4af56367925594eb932b1f7a259ba2212dd Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:17:44 +0000 Subject: [PATCH 23/32] chore: address comments --- pkg/appgateserver/endpoint_selector.go | 2 +- pkg/appgateserver/jsonrpc.go | 6 +++++- pkg/appgateserver/relay_verifier.go | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/appgateserver/endpoint_selector.go b/pkg/appgateserver/endpoint_selector.go index 442d9fefd..380c5dad5 100644 --- a/pkg/appgateserver/endpoint_selector.go +++ b/pkg/appgateserver/endpoint_selector.go @@ -10,7 +10,7 @@ import ( ) // TODO_IMPROVE: This implements a naive greedy approach that defaults to the -// first available URL but future optimizations can be introduced. +// first available supplier. Future optimizations (e.g. Quality-of-Service) can be introduced here. // TODO(@h5law): Look into different endpoint selection depending on their suitability. // getRelayerUrl gets the URL of the relayer for the given service. func (app *appGateServer) getRelayerUrl( diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 2c8144ddc..1a5f126a5 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -14,6 +14,8 @@ import ( ) // handleJSONRPCRelay handles JSON RPC relay requests. +// It does everything from preparing, signing and sending the request. +// It then blocks on the response to come back and forward it to the provided writer. func (app *appGateServer) handleJSONRPCRelay( ctx context.Context, appAddress, serviceId string, @@ -46,7 +48,7 @@ func (app *appGateServer) handleJSONRPCRelay( relayRequest := &types.RelayRequest{ Meta: &types.RelayRequestMetadata{ SessionHeader: session.Header, - Signature: nil, + Signature: nil, // signature added below }, Payload: relayRequestPayload, } @@ -108,6 +110,8 @@ func (app *appGateServer) handleJSONRPCRelay( // the getRelayerUrl function since this is the address we are expecting to sign the response. // TODO_TECHDEBT: if the RelayResponse is an internal error response, we should not verify the signature // as in some relayer early failures, it may not be signed by the supplier. + // TODO_IMPROVE: Add more logging & telemetry so we can get visibility and signal into + // failed responses. if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { return err } diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 76432ac64..3667507d5 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -40,9 +40,8 @@ func (app *appGateServer) verifyResponse( return nil } -// getSupplierPubKeyFromAddress gets the supplier's public key from the cache or queries -// if it is not found. -// The public key is then cached before being returned. +// getSupplierPubKeyFromAddress gets the supplier's public key from the cache or +// queries if it is not found. The public key is then cached before being returned. func (app *appGateServer) getSupplierPubKeyFromAddress( ctx context.Context, supplierAddress string, @@ -52,7 +51,8 @@ func (app *appGateServer) getSupplierPubKeyFromAddress( return pubKey, nil } - // Query for the supplier account to get the application's public key to verify the relay request signature. + // Query for the supplier account to get the application's public key + // to verify the relay request signature. accQueryReq := &accounttypes.QueryAccountRequest{Address: supplierAddress} accQueryRes, err := app.accountQuerier.Account(ctx, accQueryReq) if err != nil { From 118d26def1307ea345c55f3b3d64fe2c1a99250c Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:45:42 +0000 Subject: [PATCH 24/32] chore: update comments and naming --- go.mod | 4 ++-- pkg/appgateserver/cmd/cmd.go | 11 ----------- pkg/appgateserver/options.go | 2 +- pkg/appgateserver/relay_verifier.go | 10 +++++----- pkg/appgateserver/rings.go | 26 +++++++++++++++++--------- pkg/relayer/proxy/rings.go | 14 +++++++++++--- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index adbedcf85..da331e2f6 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/athanorlabs/go-dleq v0.1.0 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 - github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -31,7 +30,6 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -75,6 +73,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -270,6 +269,7 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index bda778da7..fa7cdbb62 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -86,17 +86,6 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to parse listening endpoint: %w", err) } - // Obtain the comet websocket endpoint from the client context. - cometWSUrl, err := url.Parse(clientCtx.NodeURI + "/websocket") - if err != nil { - return fmt.Errorf("failed to parse block query URL: %w", err) - } - cometWSUrl.Scheme = "ws" - // If the comet websocket URL is not provided, use the one from the client context. - if flagCometWebsocketUrl == "" { - flagCometWebsocketUrl = cometWSUrl.String() - } - log.Printf("INFO: Creating block client, using comet websocket URL: %s...", flagCometWebsocketUrl) // Create the block client with its dependency on the events client. diff --git a/pkg/appgateserver/options.go b/pkg/appgateserver/options.go index 33f0c90a5..187c83f84 100644 --- a/pkg/appgateserver/options.go +++ b/pkg/appgateserver/options.go @@ -4,7 +4,7 @@ import ( "net/url" ) -// WithSigningInformation sets the signing key and app address for server. +// WithSigningInformation sets the signing key and app address for the server. func WithSigningInformation(signingInfo *SigningInformation) appGateServerOption { return func(appGateServer *appGateServer) { appGateServer.signingInformation = signingInfo diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 3667507d5..712eda7f9 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -17,13 +17,13 @@ func (app *appGateServer) verifyResponse( relayResponse *types.RelayResponse, ) error { // Get the supplier's public key. - pubKey, err := app.getSupplierPubKeyFromAddress(ctx, supplierAddress) + supplierPubKey, err := app.getSupplierPubKeyFromAddress(ctx, supplierAddress) if err != nil { return err } // Extract the supplier's signature - signature := relayResponse.Meta.SupplierSignature + supplierSignature := relayResponse.Meta.SupplierSignature // Get the relay response signable bytes and hash them. responseBz, err := relayResponse.GetSignableBytes() @@ -33,7 +33,7 @@ func (app *appGateServer) verifyResponse( hash := crypto.Sha256(responseBz) // Verify the relay response signature. - if !pubKey.VerifySignature(hash, signature) { + if !supplierPubKey.VerifySignature(hash, supplierSignature) { return ErrAppGateInvalidRelayResponseSignature } @@ -46,9 +46,9 @@ func (app *appGateServer) getSupplierPubKeyFromAddress( ctx context.Context, supplierAddress string, ) (cryptotypes.PubKey, error) { - pubKey, ok := app.supplierAccountCache[supplierAddress] + supplierPubKey, ok := app.supplierAccountCache[supplierAddress] if ok { - return pubKey, nil + return supplierPubKey, nil } // Query for the supplier account to get the application's public key diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index 0acf423e8..a245c6e88 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -41,7 +41,7 @@ func (app *appGateServer) getRingSingerForAppAddress(ctx context.Context, appAdd ring, err = newRingFromPoints(points) } if err != nil { - log.Printf("ERROR: Unable to get ring for address: %s [%w]", appAddress, err) + log.Printf("ERROR: Unable to get ring for address: %s [%v]", appAddress, err) return nil, err } @@ -65,8 +65,8 @@ func newRingFromPoints(points []ringtypes.Point) (*ring.Ring, error) { return ring.NewFixedKeyRingFromPublicKeys(ring_secp256k1.NewCurve(), points) } -// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given application -// address, by querying the portal module for it's delegated pubkeys +// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given +// application address, by querying the application module for it's delegated pubkeys func (app *appGateServer) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, @@ -84,9 +84,17 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( // create a slice of addresses for the ring ringAddresses := make([]string, 0) ringAddresses = append(ringAddresses, appAddress) // app address is index 0 - ringAddresses = append(ringAddresses, appAddress) // add app address twice to make the ring size of mininmum 2 - if len(res.Application.DelegateeGatewayAddresses) > 0 { - ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) // delegatee addresses are index 1+ + if len(res.Application.DelegateeGatewayAddresses) < 1 { + // add app address twice to make the ring size of mininmum 2 + // TODO_TECHDEBT: We are adding the appAddress twice because a ring + // signature requires AT LEAST two pubKeys. When the Application has + // not delegated to any gateways, we add the application's own address + // twice. This is a HACK and should be investigated as to what is the + // best approach to take in this situation. + ringAddresses = append(ringAddresses, appAddress) + } else len(res.Application.DelegateeGatewayAddresses) > 0 { + // add the delegatee gateway addresses + ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) } // get the points on the secp256k1 curve for the addresses @@ -102,9 +110,9 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( return points, nil } -// addressesToPoints converts a slice of addresses to a slice of points on the secp256k1 curve -// it does so by querying the account module for the public key for each address and converting -// them to the corresponding points on the secp256k1 curve +// addressesToPoints converts a slice of addresses to a slice of points on the +// secp256k1 curve, by querying the account module for the public key for each +// address and converting them to the corresponding points on the secp256k1 curve func (app *appGateServer) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { curve := ring_secp256k1.NewCurve() points := make([]ringtypes.Point, len(addresses)) diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index 0df1183f5..cf1cd8323 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -60,9 +60,17 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( // create a slice of addresses for the ring ringAddresses := make([]string, 0) ringAddresses = append(ringAddresses, appAddress) // app address is index 0 - ringAddresses = append(ringAddresses, appAddress) // add app address twice to make the ring size of mininmum 2 - if len(res.Application.DelegateeGatewayAddresses) > 0 { - ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) // delegatee addresses are index 1+ + if len(res.Application.DelegateeGatewayAddresses) < 1 { + // add app address twice to make the ring size of mininmum 2 + // TODO_TECHDEBT: We are adding the appAddress twice because a ring + // signature requires AT LEAST two pubKeys. When the Application has + // not delegated to any gateways, we add the application's own address + // twice. This is a HACK and should be investigated as to what is the + // best approach to take in this situation. + ringAddresses = append(ringAddresses, appAddress) + } else len(res.Application.DelegateeGatewayAddresses) > 0 { + // add the delegatee gateway addresses + ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) } // get the points on the secp256k1 curve for the addresses From ce913718d7c92fe254be6c4873c78684dc50b381 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:46:53 +0000 Subject: [PATCH 25/32] chore: fix missing if --- go.mod | 4 ++-- pkg/appgateserver/rings.go | 2 +- pkg/relayer/proxy/rings.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index da331e2f6..adbedcf85 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/athanorlabs/go-dleq v0.1.0 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 + github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -30,6 +31,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -73,7 +75,6 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect - github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -269,7 +270,6 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index a245c6e88..680203bb5 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -92,7 +92,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( // twice. This is a HACK and should be investigated as to what is the // best approach to take in this situation. ringAddresses = append(ringAddresses, appAddress) - } else len(res.Application.DelegateeGatewayAddresses) > 0 { + } else if len(res.Application.DelegateeGatewayAddresses) > 0 { // add the delegatee gateway addresses ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) } diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index cf1cd8323..9b48e6d1b 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -68,7 +68,7 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( // twice. This is a HACK and should be investigated as to what is the // best approach to take in this situation. ringAddresses = append(ringAddresses, appAddress) - } else len(res.Application.DelegateeGatewayAddresses) > 0 { + } else if len(res.Application.DelegateeGatewayAddresses) > 0 { // add the delegatee gateway addresses ringAddresses = append(ringAddresses, res.Application.DelegateeGatewayAddresses...) } From 969bf0fe1682641819905a2f0def0e9884cb58e9 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 21:54:14 +0000 Subject: [PATCH 26/32] chore: add signed relay received debug log --- pkg/appgateserver/jsonrpc.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 1a5f126a5..3be01cca7 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -88,7 +88,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Perform the HTTP request to the relayer. - log.Printf("DEBUG: Sending relay request to %s", supplierUrl) + log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err @@ -112,6 +112,7 @@ func (app *appGateServer) handleJSONRPCRelay( // as in some relayer early failures, it may not be signed by the supplier. // TODO_IMPROVE: Add more logging & telemetry so we can get visibility and signal into // failed responses. + log.Println("DEBUG: Verifying signed relay response from...") if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { return err } From 211df7ff6223c64ba6a0a6b0ce8b612d50873269 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:00:33 +0000 Subject: [PATCH 27/32] chore: cleanup comments --- pkg/appgateserver/server.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index f01a876b7..c80a6b1c6 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -26,12 +26,12 @@ type SigningInformation struct { SigningKey ringtypes.Scalar // AppAddress is the address of the application that the server is serving if - // it is nil then the application address must be included in each request + // If it is nil, then the application address must be included in each request via a query parameter. AppAddress string } // appGateServer is the server that listens for application requests and relays them to the supplier. -// it is responsible for maintaining the current session for the application, signing the requests, +// It is responsible for maintaining the current session for the application, signing the requests, // and verifying the response signatures. // The appGateServer is the basis for both applications and gateways, depending on whether the application // is running their own instance of the appGateServer or they are sending requests to a gateway running an @@ -121,7 +121,7 @@ func NewAppGateServer( return app, nil } -// Start starts the application server and blocks until the context is done +// Start starts the appgate server and blocks until the context is done // or the server returns an error. func (app *appGateServer) Start(ctx context.Context) error { // Shutdown the HTTP server when the context is done. @@ -137,12 +137,12 @@ func (app *appGateServer) Start(ctx context.Context) error { return app.server.ListenAndServe() } -// Stop stops the application server and returns any error that occurred. +// Stop stops the appgate server and returns any error that occurred. func (app *appGateServer) Stop(ctx context.Context) error { return app.server.Shutdown(ctx) } -// ServeHTTP is the HTTP handler for the application server. +// ServeHTTP is the HTTP handler for the appgate server. // It captures the application request, signs it, and sends it to the supplier. // After receiving the response from the supplier, it verifies the response signature // before returning the response to the application. @@ -153,20 +153,19 @@ func (app *appGateServer) Stop(ctx context.Context) error { // // where the serviceId is the id of the service that the application is requesting // and the other (possible) path segments are the JSON RPC request path. +// TODO_TECHDEBT: Revisit the requestPath above based on the SDK that'll be exposed in the future. func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { ctx := request.Context() // Extract the serviceId from the request path. path := request.URL.Path serviceId := strings.Split(path, "/")[1] - var appAddress string - if app.signingInformation.AppAddress == "" { + // Determine the application address. + appAddress := app.signingInformation.AppAddress + if appAddress == "" { appAddress = request.URL.Query().Get("senderAddr") - } else { - appAddress = app.signingInformation.AppAddress } - if appAddress == "" { app.replyWithError(writer, ErrAppGateMissingAppAddress) log.Print("ERROR: no application address provided") From 89786c3be246ea29d8429254296cf0e508d946d0 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:04:03 +0000 Subject: [PATCH 28/32] chore: comments comments comments --- pkg/appgateserver/rings.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index 680203bb5..f97e7d84b 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -67,6 +67,7 @@ func newRingFromPoints(points []ringtypes.Point) (*ring.Ring, error) { // getDelegatedPubKeysForAddress returns the ring used to sign a message for the given // application address, by querying the application module for it's delegated pubkeys +// and converting them to points on the secp256k1 curve in order to create the ring. func (app *appGateServer) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, @@ -86,7 +87,7 @@ func (app *appGateServer) getDelegatedPubKeysForAddress( ringAddresses = append(ringAddresses, appAddress) // app address is index 0 if len(res.Application.DelegateeGatewayAddresses) < 1 { // add app address twice to make the ring size of mininmum 2 - // TODO_TECHDEBT: We are adding the appAddress twice because a ring + // TODO_HACK: We are adding the appAddress twice because a ring // signature requires AT LEAST two pubKeys. When the Application has // not delegated to any gateways, we add the application's own address // twice. This is a HACK and should be investigated as to what is the From b6f7ef58e2ba3992ce406aae11f918ff7c278f3f Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:11:03 +0000 Subject: [PATCH 29/32] chore: comments comments comments --- pkg/relayer/proxy/rings.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index 9b48e6d1b..eeb7321ce 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -41,8 +41,9 @@ func newRingFromPoints(points []ringtypes.Point) (*ring.Ring, error) { return ring.NewFixedKeyRingFromPublicKeys(ring_secp256k1.NewCurve(), points) } -// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given application -// address, by querying the portal module for it's delegated pubkeys +// getDelegatedPubKeysForAddress returns the ring used to sign a message for the given +// application address, by querying the application module for it's delegated pubkeys +// and converting them to points on the secp256k1 curve in order to create the ring. func (rp *relayerProxy) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, @@ -86,9 +87,9 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( return points, nil } -// addressesToPoints converts a slice of addresses to a slice of points on the secp256k1 curve -// it does so by querying the account module for the public key for each address and converting -// them to the corresponding points on the secp256k1 curve +// addressesToPoints converts a slice of addresses to a slice of points on the +// secp256k1 curve, by querying the account module for the public key for each +// address and converting them to the corresponding points on the secp256k1 curve func (rp *relayerProxy) addressesToPoints(ctx context.Context, addresses []string) ([]ringtypes.Point, error) { curve := ring_secp256k1.NewCurve() points := make([]ringtypes.Point, len(addresses)) From 1d9c233794ac73ba5afd93741aa320e7c97031f6 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:32:30 +0000 Subject: [PATCH 30/32] chore: update ring comments --- pkg/appgateserver/rings.go | 4 +++- pkg/relayer/proxy/relay_verifier.go | 2 +- pkg/relayer/proxy/rings.go | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/appgateserver/rings.go b/pkg/appgateserver/rings.go index f97e7d84b..945a5e13b 100644 --- a/pkg/appgateserver/rings.go +++ b/pkg/appgateserver/rings.go @@ -1,4 +1,6 @@ -// TODO(@h5law): Move all this logic out into a shared package +// TODO_BLOCKER(@h5law): Move all this logic out into a shared package to avoid +// the duplication of core business logic between `pkg/relayer/proxy/rings.go` +// and `pkg/appgateserver/rings.go` package appgateserver import ( diff --git a/pkg/relayer/proxy/relay_verifier.go b/pkg/relayer/proxy/relay_verifier.go index a305c18d4..e64955d55 100644 --- a/pkg/relayer/proxy/relay_verifier.go +++ b/pkg/relayer/proxy/relay_verifier.go @@ -32,7 +32,7 @@ func (rp *relayerProxy) VerifyRelayRequest( if err := ringSig.Deserialize(ring_secp256k1.NewCurve(), signature); err != nil { return sdkerrors.Wrapf( ErrRelayerProxyInvalidRelayRequestSignature, - "error deserializing signature: %v", err, + "error deserializing ring signature: %v", err, ) } diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index eeb7321ce..59a19ae70 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -1,4 +1,6 @@ -// TODO(@h5law): Move all this logic out into a shared package +// TODO_BLOCKER(@h5law): Move all this logic out into a shared package to avoid +// the duplication of core business logic between `pkg/relayer/proxy/rings.go` +// and `pkg/appgateserver/rings.go` package proxy import ( From 89688abd60b52da95e47fd677dbcce9983300e67 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:33:29 +0000 Subject: [PATCH 31/32] feat: refactor appgateserver creation with depinject supplier function and signing information handled by the creator func --- pkg/appgateserver/cmd/cmd.go | 113 ++++++++++++----------------------- pkg/appgateserver/options.go | 4 +- pkg/appgateserver/server.go | 57 +++++++++++++++++- 3 files changed, 93 insertions(+), 81 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index fa7cdbb62..21052e270 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -11,13 +11,8 @@ import ( "os/signal" "cosmossdk.io/depinject" - ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" - ringtypes "github.com/athanorlabs/go-dleq/types" cosmosclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" - "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/spf13/cobra" "github.com/pokt-network/poktroll/pkg/appgateserver" @@ -77,8 +72,18 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { ctx, cancelCtx := context.WithCancel(cmd.Context()) defer cancelCtx() - // Retrieve the client context for the chain interactions. - clientCtx := cosmosclient.GetClientContextFromCmd(cmd) + // Handle interrupts in a goroutine. + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt) + + // Block until we receive an interrupt or kill signal (OS-agnostic) + <-sigCh + log.Println("INFO: Interrupt signal received, shutting down...") + + // Signal goroutines to stop + cancelCtx() + }() // Parse the listening endpoint. listeningUrl, err := url.Parse(flagListeningEndpoint) @@ -86,72 +91,30 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to parse listening endpoint: %w", err) } - log.Printf("INFO: Creating block client, using comet websocket URL: %s...", flagCometWebsocketUrl) - - // Create the block client with its dependency on the events client. - eventsQueryClient := eventsquery.NewEventsQueryClient(flagCometWebsocketUrl) - deps := depinject.Supply(eventsQueryClient) - blockClient, err := blockclient.NewBlockClient(ctx, deps, flagCometWebsocketUrl) + // Setup the AppGate server dependencies. + appGateServerDeps, err := setupAppGateServerDependencies(cmd, ctx, flagCometWebsocketUrl) if err != nil { - return fmt.Errorf("failed to create block client: %w", err) + return fmt.Errorf("failed to setup AppGate server dependencies: %w", err) } log.Println("INFO: Creating AppGate server...") - keyRecord, err := clientCtx.Keyring.Key(flagSigningKey) - if err != nil { - return fmt.Errorf("failed to get key from keyring: %w", err) - } - - appAddress, err := keyRecord.GetAddress() - if err != nil { - return fmt.Errorf("failed to get address from key: %w", err) - } - signingAddress := "" - if flagSelfSigning { - signingAddress = appAddress.String() - } - - // Convert the key record to a private key and return the scalar - // point on the secp256k1 curve that it corresponds to. - // If the key is not a secp256k1 key, this will return an error. - signingKey, err := recordLocalToScalar(keyRecord.GetLocal()) - if err != nil { - return fmt.Errorf("failed to convert private key to scalar: %w", err) - } - signingInfo := appgateserver.SigningInformation{ - SigningKey: signingKey, - AppAddress: signingAddress, - } - // Create the AppGate server. - appGateServerDeps := depinject.Supply( - clientCtx, - blockClient, - ) - appGateServer, err := appgateserver.NewAppGateServer( appGateServerDeps, - appgateserver.WithSigningInformation(&signingInfo), + appgateserver.WithSigningInformation(&appgateserver.SigningInformation{ + // provide the name of the key to use for signing all incoming requests + SigningKeyName: flagSigningKey, + // provide whether the appgate server should sign all incoming requests + // with its own ring (for applications) or not (for gateways) + SelfSigning: flagSelfSigning, + }), appgateserver.WithListeningUrl(listeningUrl), ) if err != nil { return fmt.Errorf("failed to create AppGate server: %w", err) } - // Handle interrupts in a goroutine. - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - - // Block until we receive an interrupt or kill signal (OS-agnostic) - <-sigCh - log.Println("INFO: Interrupt signal received, shutting down...") - - // Signal goroutines to stop - cancelCtx() - }() - log.Printf("INFO: Starting AppGate server, listening on %s...", listeningUrl.String()) // Start the AppGate server. @@ -164,23 +127,21 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return nil } -// recordLocalToScalar converts the private key obtained from a -// key record to a scalar point on the secp256k1 curve -func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { - if local == nil { - return nil, fmt.Errorf("cannot extract private key from key record: nil") - } - priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) - if !ok { - return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) - } - if _, ok := priv.(*secp256k1.PrivKey); !ok { - return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) - } - crv := ring_secp256k1.NewCurve() - privKey, err := crv.DecodeToScalar(priv.Bytes()) +func setupAppGateServerDependencies(cmd *cobra.Command, ctx context.Context, cometWebsocketUrl string) (depinject.Config, error) { + // Retrieve the client context for the chain interactions. + clientCtx := cosmosclient.GetClientContextFromCmd(cmd) + + // Create the events client. + eventsQueryClient := eventsquery.NewEventsQueryClient(flagCometWebsocketUrl) + + // Create the block client. + log.Printf("INFO: Creating block client, using comet websocket URL: %s...", flagCometWebsocketUrl) + deps := depinject.Supply(eventsQueryClient) + blockClient, err := blockclient.NewBlockClient(ctx, deps, flagCometWebsocketUrl) if err != nil { - return nil, fmt.Errorf("failed to decode private key: %w", err) + return nil, fmt.Errorf("failed to create block client: %w", err) } - return privKey, nil + + // Return the dependencie config. + return depinject.Supply(clientCtx, blockClient), nil } diff --git a/pkg/appgateserver/options.go b/pkg/appgateserver/options.go index 187c83f84..fb164029b 100644 --- a/pkg/appgateserver/options.go +++ b/pkg/appgateserver/options.go @@ -4,14 +4,14 @@ import ( "net/url" ) -// WithSigningInformation sets the signing key and app address for the server. +// WithSigningInformation sets the signing information for the appgate server. func WithSigningInformation(signingInfo *SigningInformation) appGateServerOption { return func(appGateServer *appGateServer) { appGateServer.signingInformation = signingInfo } } -// WithListeningUrl sets the listening URL for the app gate server. +// WithListeningUrl sets the listening URL for the appgate server. func WithListeningUrl(listeningUrl *url.URL) appGateServerOption { return func(appGateServer *appGateServer) { appGateServer.listeningEndpoint = listeningUrl diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index c80a6b1c6..1fcd5b027 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -2,6 +2,7 @@ package appgateserver import ( "context" + "fmt" "log" "net/http" "net/url" @@ -9,8 +10,11 @@ import ( "sync" "cosmossdk.io/depinject" + ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" sdkclient "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" @@ -21,6 +25,13 @@ import ( ) type SigningInformation struct { + // SelfSigning indicates whether the server is running in self-signing mode + SelfSigning bool + + // SigningKeyName is the name of the key in the keyring that corresponds to the + // private key used to sign relay requests. + SigningKeyName string + // SigningKey is the scalar point on the appropriate curve corresponding to the // signer's private key, and is used to sign relay requests via a ring signature SigningKey ringtypes.Scalar @@ -113,6 +124,28 @@ func NewAppGateServer( return nil, err } + keyRecord, err := app.clientCtx.Keyring.Key(app.signingInformation.SigningKeyName) + if err != nil { + return nil, fmt.Errorf("failed to get key from keyring: %w", err) + } + + appAddress, err := keyRecord.GetAddress() + if err != nil { + return nil, fmt.Errorf("failed to get address from key: %w", err) + } + if app.signingInformation.SelfSigning { + app.signingInformation.AppAddress = appAddress.String() + } + + // Convert the key record to a private key and return the scalar + // point on the secp256k1 curve that it corresponds to. + // If the key is not a secp256k1 key, this will return an error. + signingKey, err := recordLocalToScalar(keyRecord.GetLocal()) + if err != nil { + return nil, fmt.Errorf("failed to convert private key to scalar: %w", err) + } + app.signingInformation.SigningKey = signingKey + app.sessionQuerier = sessiontypes.NewQueryClient(app.clientCtx) app.accountQuerier = accounttypes.NewQueryClient(app.clientCtx) app.applicationQuerier = apptypes.NewQueryClient(app.clientCtx) @@ -218,13 +251,31 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) // validateConfig validates the appGateServer configuration. func (app *appGateServer) validateConfig() error { - if app.signingInformation == nil { - return ErrAppGateMissingSigningInformation - } if app.listeningEndpoint == nil { return ErrAppGateMissingListeningEndpoint } return nil } +// recordLocalToScalar converts the private key obtained from a +// key record to a scalar point on the secp256k1 curve +func recordLocalToScalar(local *keyring.Record_Local) (ringtypes.Scalar, error) { + if local == nil { + return nil, fmt.Errorf("cannot extract private key from key record: nil") + } + priv, ok := local.PrivKey.GetCachedValue().(cryptotypes.PrivKey) + if !ok { + return nil, fmt.Errorf("cannot extract private key from key record: %T", local.PrivKey.GetCachedValue()) + } + if _, ok := priv.(*secp256k1.PrivKey); !ok { + return nil, fmt.Errorf("unexpected private key type: %T, want %T", priv, &secp256k1.PrivKey{}) + } + crv := ring_secp256k1.NewCurve() + privKey, err := crv.DecodeToScalar(priv.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to decode private key: %w", err) + } + return privKey, nil +} + type appGateServerOption func(*appGateServer) From 683b28558c38ca4b28dfdc5ef2e94b6789ecd8d8 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:35:39 +0000 Subject: [PATCH 32/32] chore: re-add missing signing information check --- pkg/appgateserver/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index 1fcd5b027..d6410021c 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -251,6 +251,9 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) // validateConfig validates the appGateServer configuration. func (app *appGateServer) validateConfig() error { + if app.signingInformation == nil { + return ErrAppGateMissingSigningInformation + } if app.listeningEndpoint == nil { return ErrAppGateMissingListeningEndpoint }