From 29b9bdfe401658edb22d5742e7b680fe98b7d7df Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 14:36:08 +0100 Subject: [PATCH 001/127] refactor: `MapFn`s receive context arg --- pkg/client/tx/client.go | 1 + pkg/observable/channel/map.go | 26 +++++++++++++++++++++----- pkg/observable/channel/map_test.go | 7 ++++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index 1c083559b..a02b32542 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -494,6 +494,7 @@ func (tClient *txClient) goTimeoutPendingTransactions(ctx context.Context) { // - skip: A flag denoting if the event should be bypassed. A value of true // suggests the event be disregarded, progressing to the succeeding message. func (tClient *txClient) txEventFromEventBz( + _ context.Context, eitherEventBz either.Bytes, ) (eitherTxEvent either.Either[*TxEvent], skip bool) { diff --git a/pkg/observable/channel/map.go b/pkg/observable/channel/map.go index c6722e09d..f9e47f974 100644 --- a/pkg/observable/channel/map.go +++ b/pkg/observable/channel/map.go @@ -6,7 +6,7 @@ import ( "github.com/pokt-network/poktroll/pkg/observable" ) -type MapFn[S, D any] func(src S) (dst D, skip bool) +type MapFn[S, D any] func(ctx context.Context, src S) (dst D, skip bool) // Map transforms the given observable by applying the given transformFn to each // notification received from the observable. If the transformFn returns a skip @@ -15,24 +15,40 @@ type MapFn[S, D any] func(src S) (dst D, skip bool) func Map[S, D any]( ctx context.Context, srcObservable observable.Observable[S], - // TODO_CONSIDERATION: if this were variadic, it could simplify serial transformations. transformFn MapFn[S, D], ) observable.Observable[D] { dstObservable, dstProducer := NewObservable[D]() srcObserver := srcObservable.Subscribe(ctx) + mapInternal[S, D]( + ctx, + srcObserver, + transformFn, + func(dstNotification D) { + dstProducer <- dstNotification + }, + ) + + return dstObservable +} + +func mapInternal[S, D any]( + ctx context.Context, + srcObserver observable.Observer[S], + transformFn MapFn[S, D], + publishFn func(dstNotifications D), +) { go func() { for srcNotification := range srcObserver.Ch() { - dstNotification, skip := transformFn(srcNotification) + dstNotifications, skip := transformFn(ctx, srcNotification) if skip { continue } - dstProducer <- dstNotification + publishFn(dstNotifications) } }() - return dstObservable } // MapReplay transforms the given observable by applying the given transformFn to diff --git a/pkg/observable/channel/map_test.go b/pkg/observable/channel/map_test.go index 98b9aa8ad..aa7507a71 100644 --- a/pkg/observable/channel/map_test.go +++ b/pkg/observable/channel/map_test.go @@ -39,9 +39,6 @@ func TestMap_Word_BytesToPalindrome(t *testing.T) { // set up source bytes observable bzObservable, bzPublishCh := channel.NewObservable[[]byte]() - bytesToPalindrome := func(wordBz []byte) (palindrome, bool) { - return newPalindrome(string(wordBz)), false - } // map bytes observable to palindrome observable palindromeObservable := channel.Map(ctx, bzObservable, bytesToPalindrome) @@ -97,6 +94,10 @@ func (p *palindrome) IsValid() bool { return p.forwards == (p.backwards) } +func bytesToPalindrome(_ context.Context, wordBz []byte) (palindrome, bool) { + return newPalindrome(string(wordBz)), false +} + // reverseString reverses a string, character-by-character. func reverseString(s string) string { runes := []rune(s) From b6c9c71dabcb65633c26edb71bc9e1144388853d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 16:25:13 +0100 Subject: [PATCH 002/127] chore: add `ForEach` map shorthand operator --- pkg/observable/channel/map.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/observable/channel/map.go b/pkg/observable/channel/map.go index f9e47f974..06bcc4900 100644 --- a/pkg/observable/channel/map.go +++ b/pkg/observable/channel/map.go @@ -7,6 +7,7 @@ import ( ) type MapFn[S, D any] func(ctx context.Context, src S) (dst D, skip bool) +type ForEachFn[V any] func(ctx context.Context, src V) // Map transforms the given observable by applying the given transformFn to each // notification received from the observable. If the transformFn returns a skip @@ -80,3 +81,23 @@ func MapReplay[S, D any]( return dstObservable } + +func ForEach[V any]( + ctx context.Context, + srcObservable observable.Observable[V], + forEachFn ForEachFn[V], +) { + Map( + ctx, srcObservable, + func(ctx context.Context, src V) (dst V, skip bool) { + forEachFn(ctx, src) + + // No downstream observers; MAY always skip. + return zeroValue[V](), true + }, + ) +} + +func zeroValue[T any]() (zero T) { + return zero +} From f257b465a12f7211e7c121c3424c4e3aac30cd63 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 16:24:37 +0100 Subject: [PATCH 003/127] chore: add `/pkg/observable/filter` --- pkg/observable/filter/either.go | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pkg/observable/filter/either.go diff --git a/pkg/observable/filter/either.go b/pkg/observable/filter/either.go new file mode 100644 index 000000000..4610b2b37 --- /dev/null +++ b/pkg/observable/filter/either.go @@ -0,0 +1,52 @@ +package filter + +import ( + "context" + + "github.com/pokt-network/poktroll/pkg/either" + "github.com/pokt-network/poktroll/pkg/observable" + "github.com/pokt-network/poktroll/pkg/observable/channel" +) + +func EitherError[T any]( + ctx context.Context, + eitherObservable observable.Observable[either.Either[T]], +) observable.Observable[error] { + return channel.Map( + ctx, + eitherObservable, + mapEitherError[T], + ) +} + +func EitherSuccess[T any]( + ctx context.Context, + eitherObservable observable.Observable[either.Either[T]], +) observable.Observable[T] { + return channel.Map( + ctx, + eitherObservable, + mapEitherSuccess[T], + ) +} + +func mapEitherError[T any]( + _ context.Context, + inputEither either.Either[T], +) (_ error, skip bool) { + if _, err := inputEither.ValueOrError(); err != nil { + return err, false + } + return nil, true +} + +func mapEitherSuccess[T any]( + _ context.Context, + inputEither either.Either[T], +) (_ T, skip bool) { + value, err := inputEither.ValueOrError() + if err != nil { + return value, true + } + return value, false +} From 194cee5df0d243e9d57f71a09fc77db1bf754a0c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 16:24:45 +0100 Subject: [PATCH 004/127] chore: add `/pkg/observable/logging` --- pkg/observable/logging/logging.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/observable/logging/logging.go diff --git a/pkg/observable/logging/logging.go b/pkg/observable/logging/logging.go new file mode 100644 index 000000000..8dabaaf65 --- /dev/null +++ b/pkg/observable/logging/logging.go @@ -0,0 +1,17 @@ +package logging + +import ( + "context" + "log" + + "github.com/pokt-network/poktroll/pkg/observable" + "github.com/pokt-network/poktroll/pkg/observable/channel" +) + +func LogErrors(ctx context.Context, errs observable.Observable[error]) { + channel.ForEach(ctx, errs, forEachErrorLogError) +} + +func forEachErrorLogError(_ context.Context, err error) { + log.Print(err) +} From cca55c2ddf51cc3388dd8e447cec278a6a2184b1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 16:24:22 +0100 Subject: [PATCH 005/127] chore: add `/pkg/relayer/protocol` --- pkg/relayer/protocol/block_heights.go | 37 +++++++++++++++++++++++++++ pkg/relayer/protocol/difficulty.go | 13 ++++++++++ 2 files changed, 50 insertions(+) create mode 100644 pkg/relayer/protocol/block_heights.go create mode 100644 pkg/relayer/protocol/difficulty.go diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go new file mode 100644 index 000000000..9dceb0966 --- /dev/null +++ b/pkg/relayer/protocol/block_heights.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "encoding/binary" + "log" + "math/rand" + + "github.com/pokt-network/poktroll/pkg/client" +) + +func GetCreateClaimDistributionHeight(earliestCreateClaimBlock client.Block) int64 { + earliestCreateClaimBlockHash := earliestCreateClaimBlock.Hash() + log.Printf("using earliestCreateClaimBlock %d's hash %x as randomness", earliestCreateClaimBlock.Height(), earliestCreateClaimBlockHash) + rngSeed, _ := binary.Varint(earliestCreateClaimBlockHash) + randomNumber := rand.NewSource(rngSeed).Int63() + + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // randCreateClaimBlockHeightOffset := randomNumber % (claimproofparams.GovLatestClaimSubmissionBlocksInterval - claimproofparams.GovClaimSubmissionBlocksWindow - 1) + _ = randomNumber + randCreateClaimBlockHeightOffset := int64(0) + + return earliestCreateClaimBlock.Height() + randCreateClaimBlockHeightOffset +} + +func GetSubmitProofDistributionHeight(earliestSubmitProofBlock client.Block) int64 { + earliestSubmitProofBlockHash := earliestSubmitProofBlock.Hash() + log.Printf("using earliestSubmitProofBlock %d's hash %x as randomness", earliestSubmitProofBlock.Height(), earliestSubmitProofBlockHash) + rngSeed, _ := binary.Varint(earliestSubmitProofBlockHash) + randomNumber := rand.NewSource(rngSeed).Int63() + + // TODO_TECHDEBT: query the on-chain governance parameter once available. + //randSubmitProofBlockHeightOffset := randomNumber % (claimproofparams.GovLatestProofSubmissionBlocksInterval - claimproofparams.GovProofSubmissionBlocksWindow - 1) + _ = randomNumber + randSubmitProofBlockHeightOffset := int64(0) + + return earliestSubmitProofBlock.Height() + randSubmitProofBlockHeightOffset +} diff --git a/pkg/relayer/protocol/difficulty.go b/pkg/relayer/protocol/difficulty.go new file mode 100644 index 000000000..ad9969f91 --- /dev/null +++ b/pkg/relayer/protocol/difficulty.go @@ -0,0 +1,13 @@ +package protocol + +import ( + "encoding/hex" + "strings" +) + +func BytesDifficultyGreaterThan(bz []byte, compDifficultyBytes int) bool { + hexZerosPrefix := strings.Repeat("0", compDifficultyBytes*2) // 2 hex chars per byte. + hexBz := hex.EncodeToString(bz) + + return strings.HasPrefix(hexBz, hexZerosPrefix) +} From c6665425019b3c5e27bcd1b2553573087535149c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 14:33:21 +0100 Subject: [PATCH 006/127] chore: add `Miner` interface --- pkg/relayer/interface.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 5d3f99c2c..1a3f0db24 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -7,10 +7,20 @@ import ( "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/x/service/types" + servicetypes "github.com/pokt-network/poktroll/x/service/types" sessiontypes "github.com/pokt-network/poktroll/x/session/types" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) +type Miner interface { + MineRelays( + ctx context.Context, + servedRelays observable.Observable[servicetypes.Relay], + ) +} + +type MinerOption func(Miner) + // RelayerProxy is the interface for the proxy that serves relays to the application. // It is responsible for starting and stopping all supported RelayServers. // While handling requests and responding in a closed loop, it also notifies From ea8d8483f27dff68d5b0075423eae28efb6c5b49 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 14:33:35 +0100 Subject: [PATCH 007/127] feat: add `Miner` implementation --- pkg/either/types.go | 7 +- pkg/relayer/miner/miner.go | 372 +++++++++++++++++++++++++++++++++++++ 2 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 pkg/relayer/miner/miner.go diff --git a/pkg/either/types.go b/pkg/either/types.go index ae7092479..4f5f53f00 100644 --- a/pkg/either/types.go +++ b/pkg/either/types.go @@ -1,9 +1,12 @@ package either +import "github.com/pokt-network/poktroll/pkg/relayer" + type ( // AsyncError represents a value which could either be a synchronous error or // an asynchronous error (sent through a channel). It wraps the more generic // `Either` type specific for error channels. - AsyncError Either[chan error] - Bytes = Either[[]byte] + AsyncError Either[chan error] + Bytes = Either[[]byte] + SessionTree = Either[relayer.SessionTree] ) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go new file mode 100644 index 000000000..ae10b2074 --- /dev/null +++ b/pkg/relayer/miner/miner.go @@ -0,0 +1,372 @@ +package miner + +import ( + "context" + "crypto/sha256" + "hash" + "log" + + "cosmossdk.io/depinject" + + "github.com/pokt-network/poktroll/pkg/client" + "github.com/pokt-network/poktroll/pkg/either" + "github.com/pokt-network/poktroll/pkg/observable" + "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/observable/filter" + "github.com/pokt-network/poktroll/pkg/observable/logging" + "github.com/pokt-network/poktroll/pkg/relayer" + "github.com/pokt-network/poktroll/pkg/relayer/protocol" + servicetypes "github.com/pokt-network/poktroll/x/service/types" +) + +var ( + _ relayer.Miner = (*miner)(nil) + defaultHasher = sha256.New() + // TODO_THIS_COMMIT: where (on-chain) should this come from? + // TODO_TECHDEBT: setting this to 0 to effectively disable mining for now. + // I.e. all relays are added to the tree. + defaultRelayDifficulty = 0 +) + +// TODO: https://stackoverflow.com/questions/77190071/golang-best-practice-for-functions-intended-to-be-called-as-goroutines + +// TODO_THIS_COMMIT: define interface & unexport +// TODO_COMMENT: Explain what the responsibility of this structure is, how its used throughout +// and leave comments alongside each field. +type miner struct { + hasher hash.Hash + relayDifficulty int + + // Injected dependencies + sessionManager relayer.RelayerSessionsManager + supplierClient client.SupplierClient + blockClient client.BlockClient +} + +type minedRelay struct { + servicetypes.Relay + bytes []byte + hash []byte +} + +func NewMiner( + deps depinject.Config, + opts ...relayer.MinerOption, +) (*miner, error) { + mnr := &miner{} + + if err := depinject.Inject( + deps, + &mnr.sessionManager, + &mnr.supplierClient, + &mnr.blockClient, + ); err != nil { + return nil, err + } + + for _, opt := range opts { + opt(mnr) + } + + if err := mnr.validateConfigAndSetDefaults(); err != nil { + return nil, err + } + + return mnr, nil +} + +// MineRelays assigns the servedRelays and sessions observables & starts their respective consumer goroutines. +func (mnr *miner) MineRelays( + ctx context.Context, + servedRelays observable.Observable[servicetypes.Relay], +) { + // sessiontypes.Relay ==> either.Either[minedRelay] + eitherMinedRelays := mnr.mineRelays(ctx, servedRelays) + + // either.Either[minedRelay] ==> error + miningErrors := mnr.addReplayToSessionTree(ctx, eitherMinedRelays) + logging.LogErrors(ctx, miningErrors) + + claimedSessions := mnr.createClaims(ctx) + + mnr.submitProofs(ctx, claimedSessions) +} + +func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { + // relayer.SessionTree ==> either.SessionTree + sessionsWithOpenClaimWindow := mnr.waitForOpenClaimWindow(ctx, mnr.sessionManager.SessionsToClaim()) + + failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := + channel.NewObservable[relayer.SessionTree]() + + // either.SessionTree ==> either.SessionTree + eitherClaimedSessions := channel.Map( + ctx, sessionsWithOpenClaimWindow, + mnr.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), + ) + + // TODO_TECHDEBT: pass failed create claim sessions to some retry mechanism. + _ = failedCreateClaimSessions + logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessions)) + + // either.SessionTree ==> relayer.SessionTree + return filter.EitherSuccess(ctx, eitherClaimedSessions) +} + +func (mnr *miner) submitProofs( + ctx context.Context, + claimedSessions observable.Observable[relayer.SessionTree], +) { + // relayer.SessionTree ==> relayer.SessionTree + sessionsWithOpenProofWindow := channel.Map(ctx, claimedSessions, mnr.mapWaitForOpenProofWindow) + + failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() + + // relayer.SessionTree ==> relayer.SessionTree + eitherProvenSessions := channel.Map( + ctx, sessionsWithOpenProofWindow, + mnr.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), + ) + + // TODO_TECHDEBT: pass failed submit proof sessions to some retry mechanism. + _ = failedSubmitProofSessions + logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessions)) +} + +func (mnr *miner) validateConfigAndSetDefaults() error { + if mnr.hasher == nil { + mnr.hasher = defaultHasher + } + return nil +} + +func (mnr *miner) mineRelays( + ctx context.Context, + servedRelays observable.Observable[servicetypes.Relay], +) observable.Observable[either.Either[minedRelay]] { + // servicetypes.Relay ==> either.Either[minedRelay] + return channel.Map(ctx, servedRelays, mnr.mapMineRelay) +} + +// TODO_THIS_COMMIT: update comment. +// mapMineRelay validates, executes, & hashes the relay. If the relay's difficulty +// is above the mining difficulty, it's inserted into SMST. +func (mnr *miner) mapMineRelay( + _ context.Context, + relay servicetypes.Relay, +) (_ either.Either[minedRelay], skip bool) { + relayBz, err := relay.Marshal() + if err != nil { + return either.Error[minedRelay](err), true + } + + // Is it correct that we need to hash the key while smst.Update() could do it + // since smst has a reference to the hasher + mnr.hasher.Write(relayBz) + relayHash := mnr.hasher.Sum(nil) + mnr.hasher.Reset() + + if !protocol.BytesDifficultyGreaterThan(relayHash, defaultRelayDifficulty) { + return either.Success(minedRelay{}), true + } + + return either.Success(minedRelay{ + Relay: relay, + bytes: relayBz, + hash: relayHash, + }), false +} + +func (mnr *miner) addReplayToSessionTree( + ctx context.Context, + eitherMinedRelays observable.Observable[either.Either[minedRelay]], +) observable.Observable[error] { + // either.Either[minedRelay] ==> error + return channel.Map(ctx, eitherMinedRelays, mnr.mapAddRelayToSessionTree) +} + +func (mnr *miner) mapAddRelayToSessionTree( + _ context.Context, + eitherRelay either.Either[minedRelay], +) (_ error, skip bool) { + // Propagate any upstream errors. + relay, err := eitherRelay.ValueOrError() + if err != nil { + return err, false + } + + // ensure the session tree exists for this relay + sessionHeader := relay.GetReq().GetMeta().GetSessionHeader() + smst, err := mnr.sessionManager.EnsureSessionTree(sessionHeader) + if err != nil { + log.Printf("failed to ensure session tree: %s\n", err) + return err, false + } + + if err := smst.Update(relay.hash, relay.bytes, 1); err != nil { + log.Printf("failed to update smt: %s\n", err) + return err, false + } + + // Skip because this map function only outputs errors. + return nil, true +} + +func (mnr *miner) waitForOpenClaimWindow( + ctx context.Context, + sessionsToClaim observable.Observable[relayer.SessionTree], +) observable.Observable[relayer.SessionTree] { + // relayer.SessionTree ==> relayer.SessionTree + return channel.Map(ctx, sessionsToClaim, mnr.mapWaitForOpenClaimWindow) +} + +func (mnr *miner) mapWaitForOpenClaimWindow( + ctx context.Context, + session relayer.SessionTree, +) (_ relayer.SessionTree, skip bool) { + mnr.waitForEarliestCreateClaimDistributionHeight( + ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), + ) + + // TODO_THIS_COMMIT: reconsider logging... + //log.Printf("currentBlock: %d, creating claim", block.Height()) + return session, false +} + +// waitForEarliestCreateClaimDistributionHeight returns the earliest and latest block heights at which +// a claim can be submitted for the current session. +// explanation of how the earliest and latest submission block height is determined is available in the +// poktroll/x/servicer/keeper/msg_server_claim.go file +func (mnr *miner) waitForEarliestCreateClaimDistributionHeight( + ctx context.Context, + sessionEndHeight int64, +) { + // TODO_TECHDEBT: refactor this logic to a shared package. + + earliestCreateClaimBlockHeight := sessionEndHeight + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // + claimproofparams.GovEarliestClaimSubmissionBlocksOffset + + // we wait for earliestCreateClaimBlockHeight to be received before proceeding since we need its hash + // to know where this servicer's claim submission window starts. + log.Printf("waiting for global earliest claim submission earliestCreateClaimBlock height: %d", earliestCreateClaimBlockHeight) + earliestCreateClaimBlock := mnr.waitForBlock(ctx, earliestCreateClaimBlockHeight) + + log.Printf("received earliest claim submission earliestCreateClaimBlock height: %d, use its hash to have a random submission for the servicer", earliestCreateClaimBlock.Height()) + + earliestClaimSubmissionDistributionHeight := protocol.GetCreateClaimDistributionHeight(earliestCreateClaimBlock) + + log.Printf("earliest claim submission earliestCreateClaimBlock height for this supplier: %d", earliestClaimSubmissionDistributionHeight) + _ = mnr.waitForBlock(ctx, earliestClaimSubmissionDistributionHeight) + + // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // latestServicerClaimSubmissionBlockHeight := earliestClaimSubmissionDistributionHeight + + // claimproofparams.GovClaimSubmissionBlocksWindow + 1 +} + +// waitForBlock blocks until the block at the given height (or greater) is +// observed as committed. +func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { + subscription := mnr.blockClient.CommittedBlocksSequence(ctx).Subscribe(ctx) + defer subscription.Unsubscribe() + + for block := range subscription.Ch() { + if block.Height() >= height { + return block + } + } + + return nil +} + +func (mnr *miner) newMapClaimSessionFn( + failedCreateClaimSessions chan<- relayer.SessionTree, +) channel.MapFn[relayer.SessionTree, either.SessionTree] { + return func( + ctx context.Context, + session relayer.SessionTree, + ) (_ either.SessionTree, skip bool) { + // this session should no longer be updated + claimRoot, err := session.Flush() + if err != nil { + // TODO_THIS_COMMIT: cleanup error handling/logging + log.Printf("failed to close tree: %s", err) + return either.Error[relayer.SessionTree](err), false + } + + sessionHeader := session.GetSessionHeader() + if err := mnr.supplierClient.CreateClaim(ctx, *sessionHeader, claimRoot); err != nil { + failedCreateClaimSessions <- session + return either.Error[relayer.SessionTree](err), false + } + + return either.Success(session), false + } +} + +func (mnr *miner) mapWaitForOpenProofWindow( + ctx context.Context, + session relayer.SessionTree, +) (_ relayer.SessionTree, skip bool) { + mnr.waitForEarliestSubmitProofDistributionHeight( + ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), + ) + + // TODO_THIS_COMMIT: reconsider logging... + //log.Printf("currentBlock: %d, submitting proof", block.Height()) + return session, false +} + +// getProofSubmissionWindow returns the earliest and latest block heights at which +// a proof can be submitted. +// explanation of how the earliest and latest submission block height is determined is available in the +// poktroll/x/servicer/keeper/msg_server_claim.go file +func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( + ctx context.Context, + createClaimHeight int64, +) { + earliestSubmitProofBlockHeight := createClaimHeight + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // + claimproofparams.GovEarliestProofSubmissionBlocksOffset + + // we wait for earliestSubmitProofBlockHeight to be received before proceeding since we need its hash + log.Printf("waiting for global earliest proof submission earliestSubmitProofBlock height: %d", earliestSubmitProofBlockHeight) + earliestSubmitProofBlock := mnr.waitForBlock(ctx, earliestSubmitProofBlockHeight) + + earliestSubmitProofDistributionHeight := protocol.GetSubmitProofDistributionHeight(earliestSubmitProofBlock) + _ = mnr.waitForBlock(ctx, earliestSubmitProofDistributionHeight) + + // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // latestServicerClaimSubmissionBlockHeight := earliestSubmitProofBlockHeight + + // claimproofparams.GovProofSubmissionBlocksWindow + 1 +} + +func (mnr *miner) newMapProveSessionFn( + failedSubmitProofSessions chan<- relayer.SessionTree, +) channel.MapFn[relayer.SessionTree, either.SessionTree] { + return func( + ctx context.Context, + session relayer.SessionTree, + ) (_ either.SessionTree, skip bool) { + latestBlock := mnr.blockClient.LatestBlock(ctx) + proof, err := session.ProveClosest(latestBlock.Hash()) + if err != nil { + return either.Error[relayer.SessionTree](err), false + } + + log.Printf("currentBlock: %d, submitting proof", latestBlock.Height()+1) + // SubmitProof ensures on-chain proof inclusion so we can safely prune the tree. + if err := mnr.supplierClient.SubmitProof( + ctx, + *session.GetSessionHeader(), + proof, + ); err != nil { + failedSubmitProofSessions <- session + return either.Error[relayer.SessionTree](err), false + } + + return either.Success(session), false + } +} From 38329bb0608b3718415035613812fc14ee243d37 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 14:33:59 +0100 Subject: [PATCH 008/127] test: `Miner` implementation --- pkg/relayer/miner/miner_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/relayer/miner/miner_test.go diff --git a/pkg/relayer/miner/miner_test.go b/pkg/relayer/miner/miner_test.go new file mode 100644 index 000000000..1ed299015 --- /dev/null +++ b/pkg/relayer/miner/miner_test.go @@ -0,0 +1,17 @@ +package miner_test + +import ( + "testing" +) + +func TestNewMiner(t *testing.T) { + //tests := []struct { + // desc string + // deps depinject.Config + // opts []relayer.MinerOption + //} { + // { + // desc: "" + // }, + //} +} From d2f9cb41a79efda9829108e11ac5885e5ff098c8 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 16:47:20 +0100 Subject: [PATCH 009/127] chore: fix comment --- pkg/relayer/protocol/block_heights.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go index 9dceb0966..631d1842b 100644 --- a/pkg/relayer/protocol/block_heights.go +++ b/pkg/relayer/protocol/block_heights.go @@ -29,7 +29,7 @@ func GetSubmitProofDistributionHeight(earliestSubmitProofBlock client.Block) int randomNumber := rand.NewSource(rngSeed).Int63() // TODO_TECHDEBT: query the on-chain governance parameter once available. - //randSubmitProofBlockHeightOffset := randomNumber % (claimproofparams.GovLatestProofSubmissionBlocksInterval - claimproofparams.GovProofSubmissionBlocksWindow - 1) + // randSubmitProofBlockHeightOffset := randomNumber % (claimproofparams.GovLatestProofSubmissionBlocksInterval - claimproofparams.GovProofSubmissionBlocksWindow - 1) _ = randomNumber randSubmitProofBlockHeightOffset := int64(0) From d502b752c6dc602da14b69d7c67324010894b0a4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 17:40:15 +0100 Subject: [PATCH 010/127] chore: add godoc comments --- pkg/relayer/interface.go | 5 +++ pkg/relayer/miner/miner.go | 84 +++++++++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 1a3f0db24..d0aa062fc 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -12,6 +12,11 @@ import ( sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) +// Miner encapsulates the following responsibilities: +// - "Mining relays": Served relays are hashed and difficulty is checked. Those +// with sufficient difficulty are added to the session SMST (tree). +// - "Creating claims": The session SMST is flushed and a claim is created on-chain. +// - "Submitting proofs": The session SMST is proven and a proof is submitted on-chain. type Miner interface { MineRelays( ctx context.Context, diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index ae10b2074..a1f3043e9 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -28,11 +28,7 @@ var ( defaultRelayDifficulty = 0 ) -// TODO: https://stackoverflow.com/questions/77190071/golang-best-practice-for-functions-intended-to-be-called-as-goroutines - -// TODO_THIS_COMMIT: define interface & unexport -// TODO_COMMENT: Explain what the responsibility of this structure is, how its used throughout -// and leave comments alongside each field. +// miner implements the relayer.Miner interface. type miner struct { hasher hash.Hash relayDifficulty int @@ -43,12 +39,15 @@ type miner struct { blockClient client.BlockClient } +// minedRelay is a wrapper around a relay that has been serialized and hashed. type minedRelay struct { servicetypes.Relay bytes []byte hash []byte } +// NewMiner creates a new miner from the given dependencies and options. It +// returns an error if it has not been sufficiently configured or supplied. func NewMiner( deps depinject.Config, opts ...relayer.MinerOption, @@ -75,7 +74,12 @@ func NewMiner( return mnr, nil } -// MineRelays assigns the servedRelays and sessions observables & starts their respective consumer goroutines. +// MineRelays kicks off relay mining by mapping the servedRelays ovservable through +// a pipeline which hashes the relay, checks if it's above the mining difficulty, +// adds it to the session tree, and then maps any errors to a new observable. +// It also starts the claim and proof pipelines which are subsequently driven by +// mapping over RelayerSessionsManager's SessionsToClaim return observable. +// It does not block as map operations run in their own goroutines. func (mnr *miner) MineRelays( ctx context.Context, servedRelays observable.Observable[servicetypes.Relay], @@ -92,6 +96,12 @@ func (mnr *miner) MineRelays( mnr.submitProofs(ctx, claimedSessions) } +// createClaims maps over the RelaySessionsManager's SessionsToClaim return +// observable. For each claim, it calculates and waits for the earliest block +// height at which it is safe to claim and does so. It then maps any errors to +// a new observable which are subsequently logged. It returns an observable of +// the successfully claimed sessions. It does not block as map operations run +// in their own goroutines. func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { // relayer.SessionTree ==> either.SessionTree sessionsWithOpenClaimWindow := mnr.waitForOpenClaimWindow(ctx, mnr.sessionManager.SessionsToClaim()) @@ -113,6 +123,11 @@ func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relaye return filter.EitherSuccess(ctx, eitherClaimedSessions) } +// submitProofs maps over the given claimedSessions observable. For each session, +// it calculates and waits for the earliest block height at which it is safe to +// submit a proof and does so. It then maps any errors to a new observable which +// are subsequently logged. It does not block as map operations run in their own +// goroutines. func (mnr *miner) submitProofs( ctx context.Context, claimedSessions observable.Observable[relayer.SessionTree], @@ -133,6 +148,8 @@ func (mnr *miner) submitProofs( logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessions)) } +// validateConfigAndSetDefaults ensures that the miner has been configured with +// a hasher and uses the default hasher if not. func (mnr *miner) validateConfigAndSetDefaults() error { if mnr.hasher == nil { mnr.hasher = defaultHasher @@ -140,6 +157,8 @@ func (mnr *miner) validateConfigAndSetDefaults() error { return nil } +// mineRelays maps over the servedRelays observable, applyging the mapMineRelay +// method to each relay. It returns an observable of the mined relays. func (mnr *miner) mineRelays( ctx context.Context, servedRelays observable.Observable[servicetypes.Relay], @@ -148,9 +167,11 @@ func (mnr *miner) mineRelays( return channel.Map(ctx, servedRelays, mnr.mapMineRelay) } -// TODO_THIS_COMMIT: update comment. -// mapMineRelay validates, executes, & hashes the relay. If the relay's difficulty -// is above the mining difficulty, it's inserted into SMST. +// mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares +// its difficulty to the minimum threshold. If the relay difficulty is sifficient, +// it returns an either populated with the minedRelay value. Otherwise, it skips +// the relay. If it encounters an error, it returns an either populated with the +// error. func (mnr *miner) mapMineRelay( _ context.Context, relay servicetypes.Relay, @@ -177,6 +198,9 @@ func (mnr *miner) mapMineRelay( }), false } +// addReplayToSessionTree maps over the eitherMinedRelays observable, applying the +// mapAddRelayToSessionTree method to each relay. It returns an observable of the +// errors encountered. func (mnr *miner) addReplayToSessionTree( ctx context.Context, eitherMinedRelays observable.Observable[either.Either[minedRelay]], @@ -185,6 +209,9 @@ func (mnr *miner) addReplayToSessionTree( return channel.Map(ctx, eitherMinedRelays, mnr.mapAddRelayToSessionTree) } +// mapAddRelayToSessionTree is intended to be used as a MapFn. It adds the relay +// to the session tree. If it encounters an error, it returns the error. Otherwise, +// it skips output (only outputs errors). func (mnr *miner) mapAddRelayToSessionTree( _ context.Context, eitherRelay either.Either[minedRelay], @@ -212,6 +239,10 @@ func (mnr *miner) mapAddRelayToSessionTree( return nil, true } +// waitForOpenClaimWindow maps over the SessionsToClaim observable, applying the +// mapWaitForOpenClaimWindow method to each session. It returns an observable of +// the sessions that are ready to be claimed which is notified when the respective +// session is ready to be claimed. func (mnr *miner) waitForOpenClaimWindow( ctx context.Context, sessionsToClaim observable.Observable[relayer.SessionTree], @@ -220,6 +251,9 @@ func (mnr *miner) waitForOpenClaimWindow( return channel.Map(ctx, sessionsToClaim, mnr.mapWaitForOpenClaimWindow) } +// mapWaitForOpenClaimWindow is intended to be used as a MapFn. It calculates and +// waits for the earliest block height at which it is safe to claim and returns +// the session when it should be claimed. func (mnr *miner) mapWaitForOpenClaimWindow( ctx context.Context, session relayer.SessionTree, @@ -227,16 +261,12 @@ func (mnr *miner) mapWaitForOpenClaimWindow( mnr.waitForEarliestCreateClaimDistributionHeight( ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), ) - - // TODO_THIS_COMMIT: reconsider logging... - //log.Printf("currentBlock: %d, creating claim", block.Height()) return session, false } -// waitForEarliestCreateClaimDistributionHeight returns the earliest and latest block heights at which -// a claim can be submitted for the current session. -// explanation of how the earliest and latest submission block height is determined is available in the -// poktroll/x/servicer/keeper/msg_server_claim.go file +// waitForEarliestCreateClaimDistributionHeight returns the earliest block height +// at which a claim can be submitted. It is calculated from the session end block +// height, on-chain governance parameters, and randomized input. func (mnr *miner) waitForEarliestCreateClaimDistributionHeight( ctx context.Context, sessionEndHeight int64, @@ -266,7 +296,7 @@ func (mnr *miner) waitForEarliestCreateClaimDistributionHeight( } // waitForBlock blocks until the block at the given height (or greater) is -// observed as committed. +// observed as having been committed. func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { subscription := mnr.blockClient.CommittedBlocksSequence(ctx).Subscribe(ctx) defer subscription.Unsubscribe() @@ -280,6 +310,9 @@ func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { return nil } +// newMapClaimSessionFn returns a new MapFn that creates a claim for the given +// session. Any session which encouters errors while creating a claim is sent +// on the failedCreateClaimSessions channel. func (mnr *miner) newMapClaimSessionFn( failedCreateClaimSessions chan<- relayer.SessionTree, ) channel.MapFn[relayer.SessionTree, either.SessionTree] { @@ -305,6 +338,10 @@ func (mnr *miner) newMapClaimSessionFn( } } +// mapWaitForOpenProofWindow maps over the claimedSessions observable, applying +// the mapWaitForOpenProofWindow method to each session. It returns an observable +// of the sessions that are ready to be proven which is notified when the respective +// session is ready to be proven. func (mnr *miner) mapWaitForOpenProofWindow( ctx context.Context, session relayer.SessionTree, @@ -312,16 +349,12 @@ func (mnr *miner) mapWaitForOpenProofWindow( mnr.waitForEarliestSubmitProofDistributionHeight( ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), ) - - // TODO_THIS_COMMIT: reconsider logging... - //log.Printf("currentBlock: %d, submitting proof", block.Height()) return session, false } -// getProofSubmissionWindow returns the earliest and latest block heights at which -// a proof can be submitted. -// explanation of how the earliest and latest submission block height is determined is available in the -// poktroll/x/servicer/keeper/msg_server_claim.go file +// waitForEarliestSubmitProofDistributionHeight returns the earliest block height +// at which a proof can be submitted. It is calculated from the session claim +// creation block height, on-chain governance parameters, and randomized input. func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( ctx context.Context, createClaimHeight int64, @@ -343,6 +376,9 @@ func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( // claimproofparams.GovProofSubmissionBlocksWindow + 1 } +// newMapProveSessionFn returns a new MapFn that submits a proof for the given +// session. Any session which encouters errors while submitting a proof is sent +// on the failedSubmitProofSessions channel. func (mnr *miner) newMapProveSessionFn( failedSubmitProofSessions chan<- relayer.SessionTree, ) channel.MapFn[relayer.SessionTree, either.SessionTree] { From 48cef8056614b7230a47c2d74a9f8b09aaad0ca9 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 20:02:10 +0100 Subject: [PATCH 011/127] feat: Add Relayer struct --- pkg/relayer/interface.go | 2 +- pkg/relayer/miner/miner.go | 18 +++++++------- pkg/relayer/proxy/proxy.go | 2 +- pkg/relayer/relayer.go | 49 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 pkg/relayer/relayer.go diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index d0aa062fc..3af354bb7 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -20,7 +20,7 @@ import ( type Miner interface { MineRelays( ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], + servedRelays observable.Observable[*servicetypes.Relay], ) } diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index a1f3043e9..380951baf 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -41,7 +41,7 @@ type miner struct { // minedRelay is a wrapper around a relay that has been serialized and hashed. type minedRelay struct { - servicetypes.Relay + *servicetypes.Relay bytes []byte hash []byte } @@ -82,13 +82,13 @@ func NewMiner( // It does not block as map operations run in their own goroutines. func (mnr *miner) MineRelays( ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], + servedRelays observable.Observable[*servicetypes.Relay], ) { // sessiontypes.Relay ==> either.Either[minedRelay] eitherMinedRelays := mnr.mineRelays(ctx, servedRelays) // either.Either[minedRelay] ==> error - miningErrors := mnr.addReplayToSessionTree(ctx, eitherMinedRelays) + miningErrors := mnr.addRelayToSessionTree(ctx, eitherMinedRelays) logging.LogErrors(ctx, miningErrors) claimedSessions := mnr.createClaims(ctx) @@ -161,20 +161,20 @@ func (mnr *miner) validateConfigAndSetDefaults() error { // method to each relay. It returns an observable of the mined relays. func (mnr *miner) mineRelays( ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], + servedRelays observable.Observable[*servicetypes.Relay], ) observable.Observable[either.Either[minedRelay]] { // servicetypes.Relay ==> either.Either[minedRelay] return channel.Map(ctx, servedRelays, mnr.mapMineRelay) } // mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares -// its difficulty to the minimum threshold. If the relay difficulty is sifficient, +// its difficulty to the minimum threshold. If the relay difficulty is sufficient, // it returns an either populated with the minedRelay value. Otherwise, it skips // the relay. If it encounters an error, it returns an either populated with the // error. func (mnr *miner) mapMineRelay( _ context.Context, - relay servicetypes.Relay, + relay *servicetypes.Relay, ) (_ either.Either[minedRelay], skip bool) { relayBz, err := relay.Marshal() if err != nil { @@ -198,10 +198,10 @@ func (mnr *miner) mapMineRelay( }), false } -// addReplayToSessionTree maps over the eitherMinedRelays observable, applying the +// addRelayToSessionTree maps over the eitherMinedRelays observable, applying the // mapAddRelayToSessionTree method to each relay. It returns an observable of the // errors encountered. -func (mnr *miner) addReplayToSessionTree( +func (mnr *miner) addRelayToSessionTree( ctx context.Context, eitherMinedRelays observable.Observable[either.Either[minedRelay]], ) observable.Observable[error] { @@ -311,7 +311,7 @@ func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { } // newMapClaimSessionFn returns a new MapFn that creates a claim for the given -// session. Any session which encouters errors while creating a claim is sent +// session. Any session which encounters errors while creating a claim is sent // on the failedCreateClaimSessions channel. func (mnr *miner) newMapClaimSessionFn( failedCreateClaimSessions chan<- relayer.SessionTree, diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 6c64516cd..13309b47c 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -102,7 +102,7 @@ func NewRelayerProxy( } // Start concurrently starts all advertised relay servers and returns an error if any of them fails to start. -// This method is blocking until all RelayServers are started. +// This method is blocking as long as all RelayServers are running. func (rp *relayerProxy) Start(ctx context.Context) error { // The provided services map is built from the supplier's on-chain advertised information, // which is a runtime parameter that can be changed by the supplier. diff --git a/pkg/relayer/relayer.go b/pkg/relayer/relayer.go new file mode 100644 index 000000000..bdad5bb1d --- /dev/null +++ b/pkg/relayer/relayer.go @@ -0,0 +1,49 @@ +package relayer + +import ( + "context" + + "cosmossdk.io/depinject" +) + +type RelayerOption func(*Relayer) + +// Relayer is the main struct that encapsulates the relayer's responsibilities. +// It starts and stops the RelayerProxy and provide the served relays observable to them miner. +type Relayer struct { + relayerProxy RelayerProxy + miner Miner +} + +// NewRelayer creates a new Relayer instance with the given dependencies. +// It injects the dependencies into the Relayer instance and returns it. +func NewRelayer( + deps depinject.Config, + opts ...RelayerOption, +) (*Relayer, error) { + rel := &Relayer{} + + if err := depinject.Inject(deps, &rel.relayerProxy, &rel.miner); err != nil { + return nil, err + } + + for _, opt := range opts { + opt(rel) + } + + return rel, nil +} + +// Start provides the miner with the served relays observable and starts the relayer proxy. +// This method is blocking while the relayer proxy is running and returns when Stop is called +// or when the relayer proxy fails to start. +func (rel *Relayer) Start(ctx context.Context) error { + rel.miner.MineRelays(ctx, rel.relayerProxy.ServedRelays()) + return rel.relayerProxy.Start(ctx) +} + +// Stop stops the relayer proxy which in turn stops all advertised relay servers +// and unsubscribes the miner from the served relays observable. +func (rel *Relayer) Stop(ctx context.Context) error { + return rel.relayerProxy.Stop(ctx) +} From cba145e67b49bae2089b0ea334f0666d28d544e5 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 22:16:30 +0100 Subject: [PATCH 012/127] chore: Rename to RelayMiner --- pkg/relayer/relayer.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/relayer/relayer.go b/pkg/relayer/relayer.go index bdad5bb1d..a5225557a 100644 --- a/pkg/relayer/relayer.go +++ b/pkg/relayer/relayer.go @@ -6,22 +6,22 @@ import ( "cosmossdk.io/depinject" ) -type RelayerOption func(*Relayer) +type RelayerOption func(*relayMiner) -// Relayer is the main struct that encapsulates the relayer's responsibilities. +// relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). // It starts and stops the RelayerProxy and provide the served relays observable to them miner. -type Relayer struct { +type relayMiner struct { relayerProxy RelayerProxy miner Miner } -// NewRelayer creates a new Relayer instance with the given dependencies. +// NewRelayMiner creates a new Relayer instance with the given dependencies. // It injects the dependencies into the Relayer instance and returns it. -func NewRelayer( +func NewRelayMiner( deps depinject.Config, opts ...RelayerOption, -) (*Relayer, error) { - rel := &Relayer{} +) (*relayMiner, error) { + rel := &relayMiner{} if err := depinject.Inject(deps, &rel.relayerProxy, &rel.miner); err != nil { return nil, err @@ -37,13 +37,14 @@ func NewRelayer( // Start provides the miner with the served relays observable and starts the relayer proxy. // This method is blocking while the relayer proxy is running and returns when Stop is called // or when the relayer proxy fails to start. -func (rel *Relayer) Start(ctx context.Context) error { - rel.miner.MineRelays(ctx, rel.relayerProxy.ServedRelays()) +func (rel *relayMiner) Start(ctx context.Context) error { + // MineRelays does not block and subscribes to the served relays observable. + rel.miner.StartMiningRelays(ctx, rel.relayerProxy.ServedRelays()) return rel.relayerProxy.Start(ctx) } // Stop stops the relayer proxy which in turn stops all advertised relay servers // and unsubscribes the miner from the served relays observable. -func (rel *Relayer) Stop(ctx context.Context) error { +func (rel *relayMiner) Stop(ctx context.Context) error { return rel.relayerProxy.Stop(ctx) } From c81c274d583d775f1e983525be67fd267376c048 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 22:18:13 +0100 Subject: [PATCH 013/127] chore: Rename relay miner file --- pkg/relayer/{relayer.go => relayminer.go} | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename pkg/relayer/{relayer.go => relayminer.go} (94%) diff --git a/pkg/relayer/relayer.go b/pkg/relayer/relayminer.go similarity index 94% rename from pkg/relayer/relayer.go rename to pkg/relayer/relayminer.go index a5225557a..6cf9bbf97 100644 --- a/pkg/relayer/relayer.go +++ b/pkg/relayer/relayminer.go @@ -23,7 +23,11 @@ func NewRelayMiner( ) (*relayMiner, error) { rel := &relayMiner{} - if err := depinject.Inject(deps, &rel.relayerProxy, &rel.miner); err != nil { + if err := depinject.Inject( + deps, + &rel.relayerProxy, + &rel.miner, + ); err != nil { return nil, err } From 0e38e7dc9a183ea88413957179f3329b1f2956b2 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 22:20:44 +0100 Subject: [PATCH 014/127] chore: Remove unused RelayerOption parameter --- pkg/relayer/relayminer.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index 6cf9bbf97..16c16f7b9 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -6,8 +6,6 @@ import ( "cosmossdk.io/depinject" ) -type RelayerOption func(*relayMiner) - // relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). // It starts and stops the RelayerProxy and provide the served relays observable to them miner. type relayMiner struct { @@ -17,10 +15,7 @@ type relayMiner struct { // NewRelayMiner creates a new Relayer instance with the given dependencies. // It injects the dependencies into the Relayer instance and returns it. -func NewRelayMiner( - deps depinject.Config, - opts ...RelayerOption, -) (*relayMiner, error) { +func NewRelayMiner(deps depinject.Config) (*relayMiner, error) { rel := &relayMiner{} if err := depinject.Inject( @@ -31,10 +26,6 @@ func NewRelayMiner( return nil, err } - for _, opt := range opts { - opt(rel) - } - return rel, nil } From f9e1cbc5b4a11c4e622d62bc9a30b74529b6898d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Thu, 9 Nov 2023 12:24:15 -0800 Subject: [PATCH 015/127] [Test] First step for automated E2E Relay test (#167) - Fixed helpers for localnet regenesis - Added an application & supplier to the genesis file - Initializing appMap & supplierMap in E2E tests - Add support for the app's codec (for unmarshaling responses) in E2E tests - Adding a placeholder for `e2e/tests/relay.feature` --- Co-authored-by: harry <53987565+h5law@users.noreply.github.com> --- Makefile | 12 ++- config.yml | 73 +++++-------- e2e/tests/init_test.go | 126 +++++++++++++++++++++- e2e/tests/node.go | 4 +- e2e/tests/relay.feature | 19 ++++ go.mod | 4 +- x/session/client/cli/query_get_session.go | 2 +- x/session/keeper/session_hydrator_test.go | 2 +- 8 files changed, 179 insertions(+), 63 deletions(-) create mode 100644 e2e/tests/relay.feature diff --git a/Makefile b/Makefile index 1860b6156..1f06b4f68 100644 --- a/Makefile +++ b/Makefile @@ -114,9 +114,9 @@ localnet_regenesis: ## Regenerate the localnet genesis file # NOTE: intentionally not using --home flag to avoid overwriting the test keyring ignite chain init mkdir -p $(POKTROLLD_HOME)/config/ - cp -r ${HOME}/.pocket/keyring-test $(POKTROLLD_HOME) - cp ${HOME}/.pocket/config/*_key.json $(POKTROLLD_HOME)/config/ - cp ${HOME}/.pocket/config/genesis.json $(POKTROLLD_HOME)/config/ + cp -r ${HOME}/.poktroll/keyring-test $(POKTROLLD_HOME) + cp ${HOME}/.poktroll/config/*_key.json $(POKTROLLD_HOME)/config/ + cp ${HOME}/.poktroll/config/genesis.json $(POKTROLLD_HOME)/config/ ############### ### Linting ### @@ -135,7 +135,9 @@ go_imports: check_go_version ## Run goimports on all go files .PHONY: test_e2e test_e2e: ## Run all E2E tests - export POCKET_NODE=$(POCKET_NODE) POKTROLLD_HOME=../../$(POKTROLLD_HOME) && go test -v ./e2e/tests/... -tags=e2e + export POCKET_NODE=$(POCKET_NODE) && \ + POKTROLLD_HOME=../../$(POKTROLLD_HOME) && \ + go test -v ./e2e/tests/... -tags=e2e .PHONY: go_test go_test: check_go_version ## Run all go tests @@ -392,7 +394,7 @@ supplier3_unstake: ## Unstake supplier3 .PHONY: get_session get_session: ## Retrieve the session given the following env vars: (APP_ADDR, SVC, HEIGHT) - pocketd --home=$(POCKETD_HOME) q session get-session $(APP) $(SVC) $(HEIGHT) --node $(POCKET_NODE) + poktrolld --home=$(POKTROLLD_HOME) q session get-session $(APP) $(SVC) $(HEIGHT) --node $(POCKET_NODE) .PHONY: get_session_app1_anvil get_session_app1_anvil: ## Retrieve the session for (app1, anvil, latest_height) diff --git a/config.yml b/config.yml index 8b235fc3f..179f60bd7 100644 --- a/config.yml +++ b/config.yml @@ -81,52 +81,27 @@ genesis: - amount: "10000" denom: upokt application: - params: - maxDelegatedGateways: 7 - applicationList: - - address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4 - stake: - amount: "1000" - denom: upokt - # TODO(@Olshansk): Update genesis to include services - # services: - # - id: svc1 - # name: TODO - # - id: svc2 - # name: TODO - # params: {} - # TODO(@Olshansk): Update genesis to include suppliers - # supplier: - # params: {} - # suppliersList: - # - address: pokt19a3t4yunp0dlpfjrp7qwnzwlrzd5fzs2gjaaaj - # services: - # - endpoints: - # - configs: - # - key: TIMEOUT - # value: 30s - # metadata: - # entries: {} - # rpc_type: WEBSOCKET - # url: ws://localhost:8546/ - # id: - # id: svc1 - # name: Pocket Network Service 1 - # metadata: - # entries: {} - # - endpoints: - # - configs: - # - key: TIMEOUT - # value: 60s - # metadata: - # entries: {} - # rpc_type: JSON_RPC - # url: http://localhost:8545 - # id: - # id: svc2 - # name: Pocket Network Service 2 - # metadata: - # entries: {} - # stake: - # amount: "1000000" - # denom: upokt + applicationList: + - address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4 + delegatee_gateway_addresses: [] + service_configs: + - service: + id: anvil + name: "" + stake: + amount: "1000" + denom: upokt + supplier: + supplierList: + - address: pokt19a3t4yunp0dlpfjrp7qwnzwlrzd5fzs2gjaaaj + services: + - endpoints: + - configs: [] + rpc_type: JSON_RPC + url: http://anvil:8547 + service: + id: anvil + name: "" + stake: + amount: "1000" + denom: upokt diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index e1284417a..6bd5aeb27 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -10,15 +10,28 @@ import ( "testing" "time" + tmcli "github.com/cometbft/cometbft/libs/cli" + "github.com/cosmos/cosmos-sdk/codec" "github.com/regen-network/gocuke" "github.com/stretchr/testify/require" + + "github.com/pokt-network/poktroll/app" + apptypes "github.com/pokt-network/poktroll/x/application/types" + sessiontypes "github.com/pokt-network/poktroll/x/session/types" + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" ) var ( - addrRe *regexp.Regexp - amountRe *regexp.Regexp - accNameToAddrMap = make(map[string]string) - keyRingFlag = "--keyring-backend=test" + addrRe *regexp.Regexp + amountRe *regexp.Regexp + + accNameToAddrMap = make(map[string]string) + accAddrToNameMap = make(map[string]string) + accNameToAppMap = make(map[string]apptypes.Application) + accNameToSupplierMap = make(map[string]sharedtypes.Supplier) + + keyRingFlag = "--keyring-backend=test" ) func init() { @@ -30,12 +43,16 @@ type suite struct { gocuke.TestingT pocketd *pocketdBin scenarioState map[string]any // temporary state for each scenario + cdc codec.Codec } func (s *suite) Before() { s.pocketd = new(pocketdBin) s.scenarioState = make(map[string]any) + s.cdc = app.MakeEncodingConfig().Marshaler s.buildAddrMap() + s.buildAppMap() + s.buildSupplierMap() } // TestFeatures runs the e2e tests specified in any .features files in this directory @@ -175,6 +192,64 @@ func (s *suite) TheForAccountIsStakedWithUpokt(actorType, accName string, amount } } +func (s *suite) TheApplicationIsStakedForService(appName string, serviceId string) { + for _, serviceConfig := range accNameToAppMap[appName].ServiceConfigs { + if serviceConfig.Service.Id == serviceId { + return + } + } + s.Fatalf("application %s is not staked for service %s", appName, serviceId) +} + +func (s *suite) TheSupplierIsStakedForService(supplierName string, serviceId string) { + for _, serviceConfig := range accNameToSupplierMap[supplierName].Services { + if serviceConfig.Service.Id == serviceId { + return + } + } + s.Fatalf("supplier %s is not staked for service %s", supplierName, serviceId) +} + +func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName string, serviceId string, supplierName string) { + app, found := accNameToAppMap[appName] + if !found { + s.Fatalf("application %s not found", appName) + } + expectedSupplier, found := accNameToSupplierMap[supplierName] + if !found { + s.Fatalf("supplier %s not found", supplierName) + } + argsAndFlags := []string{ + "query", + "session", + "get-session", + app.Address, + serviceId, + fmt.Sprintf("--%s=json", tmcli.OutputFlag), + } + res, err := s.pocketd.RunCommandOnHost("", argsAndFlags...) + if err != nil { + s.Fatalf("error getting session for app %s and service %s: %s", appName, serviceId, err) + } + var resp sessiontypes.QueryGetSessionResponse + responseBz := []byte(strings.TrimSpace(res.Stdout)) + s.cdc.MustUnmarshalJSON(responseBz, &resp) + for _, supplier := range resp.Session.Suppliers { + if supplier.Address == expectedSupplier.Address { + return + } + } + s.Fatalf("session for app %s and service %s does not contain supplier %s", appName, serviceId, supplierName) +} + +func (s *suite) TheApplicationSendsTheSupplierARelayRequestForService(appName string, supplierName string, requestName string, serviceId string) { + // TODO(#126, @Olshansk): Implement this step +} + +func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { + // TODO(#126, @Olshansk): Implement this step +} + func (s *suite) getStakedAmount(actorType, accName string) (bool, int) { s.Helper() args := []string{ @@ -216,6 +291,49 @@ func (s *suite) buildAddrMap() { name := match[2] address := match[1] accNameToAddrMap[name] = address + accAddrToNameMap[address] = name + } +} + +func (s *suite) buildAppMap() { + s.Helper() + argsAndFlags := []string{ + "query", + "application", + "list-application", + fmt.Sprintf("--%s=json", tmcli.OutputFlag), + } + res, err := s.pocketd.RunCommandOnHost("", argsAndFlags...) + if err != nil { + s.Fatalf("error getting application list: %s", err) + } + s.pocketd.result = res + var resp apptypes.QueryAllApplicationResponse + responseBz := []byte(strings.TrimSpace(res.Stdout)) + s.cdc.MustUnmarshalJSON(responseBz, &resp) + for _, app := range resp.Application { + accNameToAppMap[accAddrToNameMap[app.Address]] = app + } +} + +func (s *suite) buildSupplierMap() { + s.Helper() + argsAndFlags := []string{ + "query", + "supplier", + "list-supplier", + fmt.Sprintf("--%s=json", tmcli.OutputFlag), + } + res, err := s.pocketd.RunCommandOnHost("", argsAndFlags...) + if err != nil { + s.Fatalf("error getting supplier list: %s", err) + } + s.pocketd.result = res + var resp suppliertypes.QueryAllSupplierResponse + responseBz := []byte(strings.TrimSpace(res.Stdout)) + s.cdc.MustUnmarshalJSON(responseBz, &resp) + for _, supplier := range resp.Supplier { + accNameToSupplierMap[accAddrToNameMap[supplier.Address]] = supplier } } diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 2a8c81a73..e8ed04dc1 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -10,6 +10,8 @@ import ( "strings" ) +// TODO_TECHDEBT(https://github.com/ignite/cli/issues/3737): We're using a combination +// of `pocketd` (legacy) and `poktrolld` (current) because of an issue of how ignite works. var ( // defaultRPCURL used by pocketdBin to run remote commands defaultRPCURL = os.Getenv("POCKET_NODE") @@ -26,7 +28,7 @@ func init() { defaultRPCURL = fmt.Sprintf("tcp://%s:%d", defaultRPCHost, defaultRPCPort) } if defaultHome == "" { - defaultHome = "../../localnet/pocketd" + defaultHome = "../../localnet/poktrolld" } } diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature new file mode 100644 index 000000000..4f071bfa6 --- /dev/null +++ b/e2e/tests/relay.feature @@ -0,0 +1,19 @@ +Feature: Relay Namespace + + Scenario: App can send relay to Supplier + Given the user has the pocketd binary installed + And the application "app1" is staked for service "anvil" + And the supplier "supplier1" is staked for service "anvil" + And the session for application "app1" and service "anvil" contains the supplier "supplier1" + When the application "app1" sends the supplier "supplier1" a "getBlock" relay request for service "anvil" + Then the application "app1" receives a successful relay response signed by "supplier1" + + # TODO_TEST(@Olshansk): + # - Successful relay if using a gateway to proxy the relay + # - Succeedful relays when using multiple suppliers for app in some session + # - Successful deduction of app's balance after claim & proof lifecycle (requires querying claims, proofs, session start/end) + # - Successful inflatino of supplier's balance after claim & proof lifecycle (requires querying claims, proofs, session start/end) + # - Error if app1 is not staked for svc1 but relay is sent + # - Error if supplier is not staked for svc1 but relay is sent + # - Error if claiming the session too early + # - Error if proving the session too early \ No newline at end of file diff --git a/go.mod b/go.mod index 51aa32938..6856b606c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( cosmossdk.io/math v1.0.1 github.com/cometbft/cometbft v0.37.2 github.com/cometbft/cometbft-db v0.8.0 - github.com/cosmos/cosmos-proto v1.0.0-beta.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/cosmos/gogoproto v1.4.10 github.com/cosmos/ibc-go/v7 v7.1.0 @@ -29,7 +28,6 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.12.0 golang.org/x/sync v0.3.0 - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 google.golang.org/grpc v1.56.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -73,6 +71,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/iavl v0.20.0 // indirect @@ -268,6 +267,7 @@ require ( gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/x/session/client/cli/query_get_session.go b/x/session/client/cli/query_get_session.go index 41b441de7..9659ce331 100644 --- a/x/session/client/cli/query_get_session.go +++ b/x/session/client/cli/query_get_session.go @@ -24,7 +24,7 @@ func CmdGetSession() *cobra.Command { This is a query operation that will not result in a state transition but simply gives a view into the chain state. Example: -$ pocketd --home=$(POCKETD_HOME) q session get-session pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4 svc1 42 --node $(POCKET_NODE)`, +$ poktrolld --home=$(POKTROLLD_HOME) q session get-session pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4 svc1 42 --node $(POCKET_NODE)`, Args: cobra.RangeArgs(2, 3), RunE: func(cmd *cobra.Command, args []string) (err error) { appAddressString := args[0] diff --git a/x/session/keeper/session_hydrator_test.go b/x/session/keeper/session_hydrator_test.go index 897dd8539..5e72e1151 100644 --- a/x/session/keeper/session_hydrator_test.go +++ b/x/session/keeper/session_hydrator_test.go @@ -104,7 +104,7 @@ func TestSession_HydrateSession_Metadata(t *testing.T) { }, { desc: "blockHeight > contextHeight", - blockHeight: 9001, // block height over 9000 is too height given that the context height is 100 + blockHeight: 9001, // block height over 9000 is too high given that the context height is 100 errExpected: types.ErrSessionHydration, }, From 0e72490ba7d36c3d53b39c9e62e159c7321ba755 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 21:25:44 +0100 Subject: [PATCH 016/127] [Relayer] refactor: simplify `RelayerSessionsManager` (#169) * refactor: `MapFn`s receive context arg * feat: add `MapExpand` observable operator * refactor: `RelayerSessionsManager` to be more reactive * chore: add godoc comment * chore: review feedback improvements * trigger CI --- pkg/observable/channel/map.go | 25 ++++++++++++++++++ pkg/relayer/session/session.go | 47 ++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/pkg/observable/channel/map.go b/pkg/observable/channel/map.go index 3f9a0a25a..d6053aefa 100644 --- a/pkg/observable/channel/map.go +++ b/pkg/observable/channel/map.go @@ -33,6 +33,31 @@ func Map[S, D any]( return dstObservable } +// MapExpand transforms the given observable by applying the given transformFn to +// each notification received from the observable, similar to Map; however, the +// transformFn returns a slice of output notifications for each input notification. +func MapExpand[S, D any]( + ctx context.Context, + srcObservable observable.Observable[S], + transformFn MapFn[S, []D], +) observable.Observable[D] { + dstObservable, dstPublishCh := NewObservable[D]() + srcObserver := srcObservable.Subscribe(ctx) + + go goMapTransformNotification( + ctx, + srcObserver, + transformFn, + func(dstNotifications []D) { + for _, dstNotification := range dstNotifications { + dstPublishCh <- dstNotification + } + }, + ) + + return dstObservable +} + // MapReplay transforms the given observable by applying the given transformFn to // each notification received from the observable. If the transformFn returns a // skip bool of true, the notification is skipped and not emitted to the resulting diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index d0527858c..0fc37c0d3 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -5,7 +5,7 @@ import ( "log" "sync" - blockclient "github.com/pokt-network/poktroll/pkg/client" + "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" "github.com/pokt-network/poktroll/pkg/relayer" @@ -33,7 +33,7 @@ type relayerSessionsManager struct { sessionsTreesMu *sync.Mutex // blockClient is used to get the notifications of committed blocks. - blockClient blockclient.BlockClient + blockClient client.BlockClient // storesDirectory points to a path on disk where KVStore data files are created. storesDirectory string @@ -43,16 +43,19 @@ type relayerSessionsManager struct { func NewRelayerSessions( ctx context.Context, storesDirectory string, - blockClient blockclient.BlockClient, + blockClient client.BlockClient, ) relayer.RelayerSessionsManager { rs := &relayerSessionsManager{ sessionsTrees: make(sessionsTreesMap), storesDirectory: storesDirectory, blockClient: blockClient, } - rs.sessionsToClaim, rs.sessionsToClaimPublisher = channel.NewObservable[relayer.SessionTree]() - go rs.goListenToCommittedBlocks(ctx) + rs.sessionsToClaim = channel.MapExpand[client.Block, relayer.SessionTree]( + ctx, + blockClient.CommittedBlocksSequence(ctx), + rs.mapBlockToSessionsToClaim, + ) return rs } @@ -92,26 +95,26 @@ func (rs *relayerSessionsManager) EnsureSessionTree(sessionHeader *sessiontypes. return sessionTree, nil } -// goListenToCommittedBlocks listens to committed blocks so that rs.sessionsToClaimPublisher -// can notify when sessions are ready to be claimed. -// It is intended to be called as a background goroutine. -func (rs *relayerSessionsManager) goListenToCommittedBlocks(ctx context.Context) { - committedBlocks := rs.blockClient.CommittedBlocksSequence(ctx).Subscribe(ctx).Ch() - - for block := range committedBlocks { - // Check if there are sessions that need to enter the claim/proof phase - // as their end block height was the one before the last committed block. - // Iterate over the sessionsTrees map to get the ones that end at a block height - // lower than the current block height. - for endBlockHeight, sessionsTreesEndingAtBlockHeight := range rs.sessionsTrees { - if endBlockHeight < block.Height() { - // Iterate over the sessionsTrees that end at this block height (or less) and publish them. - for _, sessionTree := range sessionsTreesEndingAtBlockHeight { - rs.sessionsToClaimPublisher <- sessionTree - } +// mapBlockToSessionsToClaim maps a block to a list of sessions which can be +// claimed as of that block. +func (rs *relayerSessionsManager) mapBlockToSessionsToClaim( + _ context.Context, + block client.Block, +) (sessionTrees []relayer.SessionTree, skip bool) { + // Check if there are sessions that need to enter the claim/proof phase + // as their end block height was the one before the last committed block. + // Iterate over the sessionsTrees map to get the ones that end at a block height + // lower than the current block height. + for endBlockHeight, sessionsTreesEndingAtBlockHeight := range rs.sessionsTrees { + if endBlockHeight < block.Height() { + // Iterate over the sessionsTrees that end at this block height (or + // less) and add them to the list of sessionTrees to be published. + for _, sessionTree := range sessionsTreesEndingAtBlockHeight { + sessionTrees = append(sessionTrees, sessionTree) } } } + return sessionTrees, false } // removeFromRelayerSessions removes the SessionTree from the relayerSessions. From 107f6ddf13d34fba5be03241662c65729949c133 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 22:19:50 +0100 Subject: [PATCH 017/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/interface.go | 13 +++++++++---- pkg/relayer/miner/miner.go | 2 +- pkg/relayer/protocol/difficulty.go | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index d0aa062fc..d12c12722 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -13,10 +13,15 @@ import ( ) // Miner encapsulates the following responsibilities: -// - "Mining relays": Served relays are hashed and difficulty is checked. Those -// with sufficient difficulty are added to the session SMST (tree). -// - "Creating claims": The session SMST is flushed and a claim is created on-chain. -// - "Submitting proofs": The session SMST is proven and a proof is submitted on-chain. +// 1. Mining relays: Served relays are hashed and difficulty is checked. +// Those with sufficient difficulty are added to the session SMST (tree) +// to be applicable for relay volume. +// 2. Creating claims: The session SMST is flushed and an on-chain +// claim is created to the amount of work done by committing +// the tree's root. +// 3. Submitting proofs: A pseudo-random branch from the session SMST +// is "requested" (through on-chain mechanisms) and the necessary proof +// is submitted on-hcina. type Miner interface { MineRelays( ctx context.Context, diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index a1f3043e9..92ca15a9f 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -74,7 +74,7 @@ func NewMiner( return mnr, nil } -// MineRelays kicks off relay mining by mapping the servedRelays ovservable through +// MineRelays kicks off relay mining by mapping the servedRelays observable through // a pipeline which hashes the relay, checks if it's above the mining difficulty, // adds it to the session tree, and then maps any errors to a new observable. // It also starts the claim and proof pipelines which are subsequently driven by diff --git a/pkg/relayer/protocol/difficulty.go b/pkg/relayer/protocol/difficulty.go index ad9969f91..4743d546d 100644 --- a/pkg/relayer/protocol/difficulty.go +++ b/pkg/relayer/protocol/difficulty.go @@ -5,6 +5,10 @@ import ( "strings" ) +// TODO_BLOCKER: Revisit this part of the algorithm after initial TestNet Launch. +// TODO_TEST: Add extensive tests for the core relay mining business logic. +// BytesDifficultyGreaterThan determines if the bytes exceed a certain difficulty, and it +// is used to determine if a relay is volume applicable. See the spec for more details: https://github.com/pokt-network/pocket-network-protocol func BytesDifficultyGreaterThan(bz []byte, compDifficultyBytes int) bool { hexZerosPrefix := strings.Repeat("0", compDifficultyBytes*2) // 2 hex chars per byte. hexBz := hex.EncodeToString(bz) From ccad087f0c7db2cd4126214af6f59063bc664a7c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 22:20:20 +0100 Subject: [PATCH 018/127] chore: review feedback improvements --- pkg/observable/types.go | 12 +++++ pkg/relayer/interface.go | 2 +- pkg/relayer/miner/miner.go | 95 +++++++++++++++----------------------- 3 files changed, 50 insertions(+), 59 deletions(-) create mode 100644 pkg/observable/types.go diff --git a/pkg/observable/types.go b/pkg/observable/types.go new file mode 100644 index 000000000..f6ae991ba --- /dev/null +++ b/pkg/observable/types.go @@ -0,0 +1,12 @@ +package observable + +import ( + "github.com/pokt-network/poktroll/pkg/relayer" + servicetypes "github.com/pokt-network/poktroll/x/service/types" +) + +type ( + Error = Observable[error] + Relay = Observable[*servicetypes.Relay] + SessionTree = Observable[relayer.SessionTree] +) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index d12c12722..5964bdd6e 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -25,7 +25,7 @@ import ( type Miner interface { MineRelays( ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], + servedRelays observable.Observable[*servicetypes.Relay], ) } diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 92ca15a9f..5e5410fd6 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -76,19 +76,16 @@ func NewMiner( // MineRelays kicks off relay mining by mapping the servedRelays observable through // a pipeline which hashes the relay, checks if it's above the mining difficulty, -// adds it to the session tree, and then maps any errors to a new observable. +// adds it to the session tree, and then maps any errors to a new observable. // It also starts the claim and proof pipelines which are subsequently driven by // mapping over RelayerSessionsManager's SessionsToClaim return observable. // It does not block as map operations run in their own goroutines. -func (mnr *miner) MineRelays( - ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], -) { +func (mnr *miner) MineRelays(ctx context.Context, servedRelays observable.Relay) { // sessiontypes.Relay ==> either.Either[minedRelay] - eitherMinedRelays := mnr.mineRelays(ctx, servedRelays) + eitherMinedRelays := channel.Map(ctx, servedRelays, mnr.mapMineRelay) // either.Either[minedRelay] ==> error - miningErrors := mnr.addReplayToSessionTree(ctx, eitherMinedRelays) + miningErrors := channel.Map(ctx, eitherMinedRelays, mnr.mapAddRelayToSessionTree) logging.LogErrors(ctx, miningErrors) claimedSessions := mnr.createClaims(ctx) @@ -102,14 +99,21 @@ func (mnr *miner) MineRelays( // a new observable which are subsequently logged. It returns an observable of // the successfully claimed sessions. It does not block as map operations run // in their own goroutines. -func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { - // relayer.SessionTree ==> either.SessionTree - sessionsWithOpenClaimWindow := mnr.waitForOpenClaimWindow(ctx, mnr.sessionManager.SessionsToClaim()) +func (mnr *miner) createClaims(ctx context.Context) observable.SessionTree { + // Map SessionsToClaim observable to a new observable of the same type which + // is notified when the session is eligible to be claimed. + // relayer.SessionTree ==> relayer.SessionTree + sessionsWithOpenClaimWindow := channel.Map( + ctx, mnr.sessionManager.SessionsToClaim(), + mnr.mapWaitForOpenClaimWindow, + ) failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() - // either.SessionTree ==> either.SessionTree + // Map sessionsWithOpenClaimWindow to a new observable of an either type, + // populated with the session or an error, which is notified after the session + // claim has been created or an error has been encountered, respectively. eitherClaimedSessions := channel.Map( ctx, sessionsWithOpenClaimWindow, mnr.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), @@ -119,7 +123,8 @@ func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relaye _ = failedCreateClaimSessions logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessions)) - // either.SessionTree ==> relayer.SessionTree + // Map eitherClaimedSessions to a new observable of relayer.SessionTree which + // is notified when the corresponding claim creation succeeded. return filter.EitherSuccess(ctx, eitherClaimedSessions) } @@ -130,14 +135,21 @@ func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relaye // goroutines. func (mnr *miner) submitProofs( ctx context.Context, - claimedSessions observable.Observable[relayer.SessionTree], + claimedSessions observable.SessionTree, ) { - // relayer.SessionTree ==> relayer.SessionTree - sessionsWithOpenProofWindow := channel.Map(ctx, claimedSessions, mnr.mapWaitForOpenProofWindow) + // Map claimedSessions to a new observable of the same type which is notified + // when the session is eligible to be proven. + sessionsWithOpenProofWindow := channel.Map( + ctx, claimedSessions, + mnr.mapWaitForOpenProofWindow, + ) - failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() + failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := + channel.NewObservable[relayer.SessionTree]() - // relayer.SessionTree ==> relayer.SessionTree + // Map sessionsWithOpenProofWindow to a new observable of an either type, + // populated with the session or an error, which is notified after the session + // proof has been submitted or an error has been encountered, respectively. eitherProvenSessions := channel.Map( ctx, sessionsWithOpenProofWindow, mnr.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), @@ -157,16 +169,6 @@ func (mnr *miner) validateConfigAndSetDefaults() error { return nil } -// mineRelays maps over the servedRelays observable, applyging the mapMineRelay -// method to each relay. It returns an observable of the mined relays. -func (mnr *miner) mineRelays( - ctx context.Context, - servedRelays observable.Observable[servicetypes.Relay], -) observable.Observable[either.Either[minedRelay]] { - // servicetypes.Relay ==> either.Either[minedRelay] - return channel.Map(ctx, servedRelays, mnr.mapMineRelay) -} - // mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares // its difficulty to the minimum threshold. If the relay difficulty is sifficient, // it returns an either populated with the minedRelay value. Otherwise, it skips @@ -174,11 +176,11 @@ func (mnr *miner) mineRelays( // error. func (mnr *miner) mapMineRelay( _ context.Context, - relay servicetypes.Relay, -) (_ either.Either[minedRelay], skip bool) { + relay *servicetypes.Relay, +) (_ either.Either[*minedRelay], skip bool) { relayBz, err := relay.Marshal() if err != nil { - return either.Error[minedRelay](err), true + return either.Error[*minedRelay](err), true } // Is it correct that we need to hash the key while smst.Update() could do it @@ -188,33 +190,22 @@ func (mnr *miner) mapMineRelay( mnr.hasher.Reset() if !protocol.BytesDifficultyGreaterThan(relayHash, defaultRelayDifficulty) { - return either.Success(minedRelay{}), true + return either.Success[*minedRelay](nil), true } - return either.Success(minedRelay{ - Relay: relay, + return either.Success(&minedRelay{ + Relay: *relay, bytes: relayBz, hash: relayHash, }), false } -// addReplayToSessionTree maps over the eitherMinedRelays observable, applying the -// mapAddRelayToSessionTree method to each relay. It returns an observable of the -// errors encountered. -func (mnr *miner) addReplayToSessionTree( - ctx context.Context, - eitherMinedRelays observable.Observable[either.Either[minedRelay]], -) observable.Observable[error] { - // either.Either[minedRelay] ==> error - return channel.Map(ctx, eitherMinedRelays, mnr.mapAddRelayToSessionTree) -} - // mapAddRelayToSessionTree is intended to be used as a MapFn. It adds the relay // to the session tree. If it encounters an error, it returns the error. Otherwise, // it skips output (only outputs errors). func (mnr *miner) mapAddRelayToSessionTree( _ context.Context, - eitherRelay either.Either[minedRelay], + eitherRelay either.Either[*minedRelay], ) (_ error, skip bool) { // Propagate any upstream errors. relay, err := eitherRelay.ValueOrError() @@ -239,18 +230,6 @@ func (mnr *miner) mapAddRelayToSessionTree( return nil, true } -// waitForOpenClaimWindow maps over the SessionsToClaim observable, applying the -// mapWaitForOpenClaimWindow method to each session. It returns an observable of -// the sessions that are ready to be claimed which is notified when the respective -// session is ready to be claimed. -func (mnr *miner) waitForOpenClaimWindow( - ctx context.Context, - sessionsToClaim observable.Observable[relayer.SessionTree], -) observable.Observable[relayer.SessionTree] { - // relayer.SessionTree ==> relayer.SessionTree - return channel.Map(ctx, sessionsToClaim, mnr.mapWaitForOpenClaimWindow) -} - // mapWaitForOpenClaimWindow is intended to be used as a MapFn. It calculates and // waits for the earliest block height at which it is safe to claim and returns // the session when it should be claimed. @@ -311,7 +290,7 @@ func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { } // newMapClaimSessionFn returns a new MapFn that creates a claim for the given -// session. Any session which encouters errors while creating a claim is sent +// session. Any session which encouters an error while creating a claim is sent // on the failedCreateClaimSessions channel. func (mnr *miner) newMapClaimSessionFn( failedCreateClaimSessions chan<- relayer.SessionTree, From e0414728e3240d68a995cabed5aa76e49d76eeb5 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 22:25:34 +0100 Subject: [PATCH 019/127] chore: update start mining comment --- pkg/relayer/relayminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index 16c16f7b9..f4f8b6981 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -33,7 +33,7 @@ func NewRelayMiner(deps depinject.Config) (*relayMiner, error) { // This method is blocking while the relayer proxy is running and returns when Stop is called // or when the relayer proxy fails to start. func (rel *relayMiner) Start(ctx context.Context) error { - // MineRelays does not block and subscribes to the served relays observable. + // StartMiningRelays does not block, it only subscribes to the served relays observable. rel.miner.StartMiningRelays(ctx, rel.relayerProxy.ServedRelays()) return rel.relayerProxy.Start(ctx) } From 8cf07843b42c26e60ab527cd1752071c9b586fcb Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Thu, 9 Nov 2023 22:28:40 +0100 Subject: [PATCH 020/127] fix: Update Miner interface --- pkg/relayer/interface.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 3af354bb7..6a048bbaa 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -18,7 +18,7 @@ import ( // - "Creating claims": The session SMST is flushed and a claim is created on-chain. // - "Submitting proofs": The session SMST is proven and a proof is submitted on-chain. type Miner interface { - MineRelays( + StartMiningRelays( ctx context.Context, servedRelays observable.Observable[*servicetypes.Relay], ) From 85a49b7754da3b49b68cbdb46faaa9d39c52e79f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 22:34:38 +0100 Subject: [PATCH 021/127] fix: import cycle & goimports --- pkg/observable/types.go | 9 +-------- pkg/relayer/interface.go | 18 +++++++++--------- pkg/relayer/miner/miner.go | 9 ++++++--- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/pkg/observable/types.go b/pkg/observable/types.go index f6ae991ba..04df98201 100644 --- a/pkg/observable/types.go +++ b/pkg/observable/types.go @@ -1,12 +1,5 @@ package observable -import ( - "github.com/pokt-network/poktroll/pkg/relayer" - servicetypes "github.com/pokt-network/poktroll/x/service/types" -) - type ( - Error = Observable[error] - Relay = Observable[*servicetypes.Relay] - SessionTree = Observable[relayer.SessionTree] + Error = Observable[error] ) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 5964bdd6e..8f6c6cd14 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -13,15 +13,15 @@ import ( ) // Miner encapsulates the following responsibilities: -// 1. Mining relays: Served relays are hashed and difficulty is checked. -// Those with sufficient difficulty are added to the session SMST (tree) -// to be applicable for relay volume. -// 2. Creating claims: The session SMST is flushed and an on-chain -// claim is created to the amount of work done by committing -// the tree's root. -// 3. Submitting proofs: A pseudo-random branch from the session SMST -// is "requested" (through on-chain mechanisms) and the necessary proof -// is submitted on-hcina. +// 1. Mining relays: Served relays are hashed and difficulty is checked. +// Those with sufficient difficulty are added to the session SMST (tree) +// to be applicable for relay volume. +// 2. Creating claims: The session SMST is flushed and an on-chain +// claim is created to the amount of work done by committing +// the tree's root. +// 3. Submitting proofs: A pseudo-random branch from the session SMST +// is "requested" (through on-chain mechanisms) and the necessary proof +// is submitted on-hcina. type Miner interface { MineRelays( ctx context.Context, diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 5e5410fd6..71ed6d4a8 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -80,7 +80,10 @@ func NewMiner( // It also starts the claim and proof pipelines which are subsequently driven by // mapping over RelayerSessionsManager's SessionsToClaim return observable. // It does not block as map operations run in their own goroutines. -func (mnr *miner) MineRelays(ctx context.Context, servedRelays observable.Relay) { +func (mnr *miner) MineRelays( + ctx context.Context, + servedRelays observable.Observable[*servicetypes.Relay], +) { // sessiontypes.Relay ==> either.Either[minedRelay] eitherMinedRelays := channel.Map(ctx, servedRelays, mnr.mapMineRelay) @@ -99,7 +102,7 @@ func (mnr *miner) MineRelays(ctx context.Context, servedRelays observable.Relay) // a new observable which are subsequently logged. It returns an observable of // the successfully claimed sessions. It does not block as map operations run // in their own goroutines. -func (mnr *miner) createClaims(ctx context.Context) observable.SessionTree { +func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { // Map SessionsToClaim observable to a new observable of the same type which // is notified when the session is eligible to be claimed. // relayer.SessionTree ==> relayer.SessionTree @@ -135,7 +138,7 @@ func (mnr *miner) createClaims(ctx context.Context) observable.SessionTree { // goroutines. func (mnr *miner) submitProofs( ctx context.Context, - claimedSessions observable.SessionTree, + claimedSessions observable.Observable[relayer.SessionTree], ) { // Map claimedSessions to a new observable of the same type which is notified // when the session is eligible to be proven. From 0788e1dad1ab494f08b7fd7f34b8baedf4afa7af Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 23:16:54 +0100 Subject: [PATCH 022/127] chore: review feedback improvements --- pkg/relayer/interface.go | 2 +- pkg/relayer/miner/miner.go | 53 ++++++++++++++++++--------- pkg/relayer/miner/miner_test.go | 11 +----- pkg/relayer/protocol/block_heights.go | 26 ++++++++----- 4 files changed, 55 insertions(+), 37 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 8f6c6cd14..a8ecbae20 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -21,7 +21,7 @@ import ( // the tree's root. // 3. Submitting proofs: A pseudo-random branch from the session SMST // is "requested" (through on-chain mechanisms) and the necessary proof -// is submitted on-hcina. +// is submitted on-chain. type Miner interface { MineRelays( ctx context.Context, diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 71ed6d4a8..b53c34eb2 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -1,3 +1,19 @@ +// Package miner encapsulates the responsibilities of the relayer miner interface: +// 1. Mining relays: Served relays are hashed and difficulty is checked. +// Those with sufficient difficulty are added to the session SMST (tree) +// to be applicable for relay volume. +// 2. Creating claims: The session SMST is flushed and an on-chain +// claim is created to the amount of work done by committing +// the tree's root. +// 3. Submitting proofs: A pseudo-random branch from the session SMST +// is "requested" (through on-chain mechanisms) and the necessary proof +// is submitted on-chain. +// +// This is largely accomplished by pipelining observables of relays and sessions +// Through a series of map operations. +// +// TODO_TECHDEBT: add architecture diagrams covering observable flows throughout +// the miner package. package miner import ( @@ -22,9 +38,9 @@ import ( var ( _ relayer.Miner = (*miner)(nil) defaultHasher = sha256.New() - // TODO_THIS_COMMIT: where (on-chain) should this come from? - // TODO_TECHDEBT: setting this to 0 to effectively disable mining for now. - // I.e. all relays are added to the tree. + // TODO_BLOCKER: query on-chain governance params once available. + // Setting this to 0 to effectively disable mining for now. + // I.e., all relays are added to the tree. defaultRelayDifficulty = 0 ) @@ -240,40 +256,41 @@ func (mnr *miner) mapWaitForOpenClaimWindow( ctx context.Context, session relayer.SessionTree, ) (_ relayer.SessionTree, skip bool) { - mnr.waitForEarliestCreateClaimDistributionHeight( + mnr.waitForEarliestCreateClaimHeight( ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), ) return session, false } -// waitForEarliestCreateClaimDistributionHeight returns the earliest block height -// at which a claim can be submitted. It is calculated from the session end block -// height, on-chain governance parameters, and randomized input. -func (mnr *miner) waitForEarliestCreateClaimDistributionHeight( +// waitForEarliestCreateClaimHeight returns the earliest block height at which a +// claim can be created. It is calculated from the session end block height, +// on-chain governance parameters, and randomized input. +func (mnr *miner) waitForEarliestCreateClaimHeight( ctx context.Context, sessionEndHeight int64, ) { // TODO_TECHDEBT: refactor this logic to a shared package. - earliestCreateClaimBlockHeight := sessionEndHeight + createClaimWindowStartHeight := sessionEndHeight // TODO_TECHDEBT: query the on-chain governance parameter once available. // + claimproofparams.GovEarliestClaimSubmissionBlocksOffset - // we wait for earliestCreateClaimBlockHeight to be received before proceeding since we need its hash + // we wait for createClaimWindowStartHeight to be received before proceeding since we need its hash // to know where this servicer's claim submission window starts. - log.Printf("waiting for global earliest claim submission earliestCreateClaimBlock height: %d", earliestCreateClaimBlockHeight) - earliestCreateClaimBlock := mnr.waitForBlock(ctx, earliestCreateClaimBlockHeight) + log.Printf("waiting for global earliest claim submission createClaimWindowStartBlock height: %d", createClaimWindowStartHeight) + createClaimWindowStartBlock := mnr.waitForBlock(ctx, createClaimWindowStartHeight) - log.Printf("received earliest claim submission earliestCreateClaimBlock height: %d, use its hash to have a random submission for the servicer", earliestCreateClaimBlock.Height()) + log.Printf("received earliest claim submission createClaimWindowStartBlock height: %d, use its hash to have a random submission for the servicer", createClaimWindowStartBlock.Height()) - earliestClaimSubmissionDistributionHeight := protocol.GetCreateClaimDistributionHeight(earliestCreateClaimBlock) + earliestCreateClaimHeight := + protocol.GetEarliestCreateClaimHeight(createClaimWindowStartBlock) - log.Printf("earliest claim submission earliestCreateClaimBlock height for this supplier: %d", earliestClaimSubmissionDistributionHeight) - _ = mnr.waitForBlock(ctx, earliestClaimSubmissionDistributionHeight) + log.Printf("earliest claim submission createClaimWindowStartBlock height for this supplier: %d", earliestCreateClaimHeight) + _ = mnr.waitForBlock(ctx, earliestCreateClaimHeight) // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. // TODO_TECHDEBT: query the on-chain governance parameter once available. - // latestServicerClaimSubmissionBlockHeight := earliestClaimSubmissionDistributionHeight + + // latestServicerClaimSubmissionBlockHeight := earliestCreateClaimHeight + // claimproofparams.GovClaimSubmissionBlocksWindow + 1 } @@ -349,7 +366,7 @@ func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( log.Printf("waiting for global earliest proof submission earliestSubmitProofBlock height: %d", earliestSubmitProofBlockHeight) earliestSubmitProofBlock := mnr.waitForBlock(ctx, earliestSubmitProofBlockHeight) - earliestSubmitProofDistributionHeight := protocol.GetSubmitProofDistributionHeight(earliestSubmitProofBlock) + earliestSubmitProofDistributionHeight := protocol.GetEarliestSubmitProofHeight(earliestSubmitProofBlock) _ = mnr.waitForBlock(ctx, earliestSubmitProofDistributionHeight) // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. diff --git a/pkg/relayer/miner/miner_test.go b/pkg/relayer/miner/miner_test.go index 1ed299015..c362005bd 100644 --- a/pkg/relayer/miner/miner_test.go +++ b/pkg/relayer/miner/miner_test.go @@ -4,14 +4,7 @@ import ( "testing" ) +// TODO_TECHDEBT(@bryanchriswhite): add all the test coverage... func TestNewMiner(t *testing.T) { - //tests := []struct { - // desc string - // deps depinject.Config - // opts []relayer.MinerOption - //} { - // { - // desc: "" - // }, - //} + t.Skip("TODO_TECHDEBT(@bryanchriswhite): add all the test coverage...") } diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go index 631d1842b..a84d7dc14 100644 --- a/pkg/relayer/protocol/block_heights.go +++ b/pkg/relayer/protocol/block_heights.go @@ -8,10 +8,14 @@ import ( "github.com/pokt-network/poktroll/pkg/client" ) -func GetCreateClaimDistributionHeight(earliestCreateClaimBlock client.Block) int64 { - earliestCreateClaimBlockHash := earliestCreateClaimBlock.Hash() - log.Printf("using earliestCreateClaimBlock %d's hash %x as randomness", earliestCreateClaimBlock.Height(), earliestCreateClaimBlockHash) - rngSeed, _ := binary.Varint(earliestCreateClaimBlockHash) +// GetEarliestCreateClaimHeight returns the earliest block height at which a claim +// for a session with the given createClaimWindowStartHeight can be created. +// +// TODO_TEST(@bryanchriswhite): Add test coverage +func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int64 { + createClaimWindowStartBlockHash := createClaimWindowStartBlock.Hash() + log.Printf("using createClaimWindowStartBlock %d's hash %x as randomness", createClaimWindowStartBlock.Height(), createClaimWindowStartBlockHash) + rngSeed, _ := binary.Varint(createClaimWindowStartBlockHash) randomNumber := rand.NewSource(rngSeed).Int63() // TODO_TECHDEBT: query the on-chain governance parameter once available. @@ -19,12 +23,16 @@ func GetCreateClaimDistributionHeight(earliestCreateClaimBlock client.Block) int _ = randomNumber randCreateClaimBlockHeightOffset := int64(0) - return earliestCreateClaimBlock.Height() + randCreateClaimBlockHeightOffset + return createClaimWindowStartBlock.Height() + randCreateClaimBlockHeightOffset } -func GetSubmitProofDistributionHeight(earliestSubmitProofBlock client.Block) int64 { - earliestSubmitProofBlockHash := earliestSubmitProofBlock.Hash() - log.Printf("using earliestSubmitProofBlock %d's hash %x as randomness", earliestSubmitProofBlock.Height(), earliestSubmitProofBlockHash) +// GetEarliestSubmitProofHeight returns the earliest block height at which a proof +// for a session with the given submitProofWindowStartHeight can be submitted. +// +// TODO_TEST(@bryanchriswhite): Add test coverage. +func GetEarliestSubmitProofHeight(submitProofWindowStartBlock client.Block) int64 { + earliestSubmitProofBlockHash := submitProofWindowStartBlock.Hash() + log.Printf("using submitProofWindowStartBlock %d's hash %x as randomness", submitProofWindowStartBlock.Height(), earliestSubmitProofBlockHash) rngSeed, _ := binary.Varint(earliestSubmitProofBlockHash) randomNumber := rand.NewSource(rngSeed).Int63() @@ -33,5 +41,5 @@ func GetSubmitProofDistributionHeight(earliestSubmitProofBlock client.Block) int _ = randomNumber randSubmitProofBlockHeightOffset := int64(0) - return earliestSubmitProofBlock.Height() + randSubmitProofBlockHeightOffset + return submitProofWindowStartBlock.Height() + randSubmitProofBlockHeightOffset } From e6a558b93011b14beab2f7bc9462a8c1d3d0d9b2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Nov 2023 23:18:24 +0100 Subject: [PATCH 023/127] chore: cleanup TODO_THIS_COMMIT comments --- pkg/relayer/miner/miner.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index b53c34eb2..f1f5f7ef3 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -287,11 +287,6 @@ func (mnr *miner) waitForEarliestCreateClaimHeight( log.Printf("earliest claim submission createClaimWindowStartBlock height for this supplier: %d", earliestCreateClaimHeight) _ = mnr.waitForBlock(ctx, earliestCreateClaimHeight) - - // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. - // TODO_TECHDEBT: query the on-chain governance parameter once available. - // latestServicerClaimSubmissionBlockHeight := earliestCreateClaimHeight + - // claimproofparams.GovClaimSubmissionBlocksWindow + 1 } // waitForBlock blocks until the block at the given height (or greater) is @@ -322,8 +317,6 @@ func (mnr *miner) newMapClaimSessionFn( // this session should no longer be updated claimRoot, err := session.Flush() if err != nil { - // TODO_THIS_COMMIT: cleanup error handling/logging - log.Printf("failed to close tree: %s", err) return either.Error[relayer.SessionTree](err), false } @@ -368,11 +361,6 @@ func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( earliestSubmitProofDistributionHeight := protocol.GetEarliestSubmitProofHeight(earliestSubmitProofBlock) _ = mnr.waitForBlock(ctx, earliestSubmitProofDistributionHeight) - - // TODO_THIS_COMMIT: this didn't seem to be used, confirm and remove. - // TODO_TECHDEBT: query the on-chain governance parameter once available. - // latestServicerClaimSubmissionBlockHeight := earliestSubmitProofBlockHeight + - // claimproofparams.GovProofSubmissionBlocksWindow + 1 } // newMapProveSessionFn returns a new MapFn that submits a proof for the given From 9c75b2c706702c903b6d40b10f62643e95744692 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 10:38:26 +0100 Subject: [PATCH 024/127] chore: improve var & func names for clarity and consistency --- pkg/relayer/miner/miner.go | 57 ++++++++++++++------------- pkg/relayer/protocol/block_heights.go | 12 +++--- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index f1f5f7ef3..6c4369e2f 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -124,7 +124,7 @@ func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relaye // relayer.SessionTree ==> relayer.SessionTree sessionsWithOpenClaimWindow := channel.Map( ctx, mnr.sessionManager.SessionsToClaim(), - mnr.mapWaitForOpenClaimWindow, + mnr.mapWaitForEarliestCreateClaimHeight, ) failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := @@ -160,7 +160,7 @@ func (mnr *miner) submitProofs( // when the session is eligible to be proven. sessionsWithOpenProofWindow := channel.Map( ctx, claimedSessions, - mnr.mapWaitForOpenProofWindow, + mnr.mapWaitForEarliestSubmitProofHeight, ) failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := @@ -249,10 +249,11 @@ func (mnr *miner) mapAddRelayToSessionTree( return nil, true } -// mapWaitForOpenClaimWindow is intended to be used as a MapFn. It calculates and -// waits for the earliest block height at which it is safe to claim and returns -// the session when it should be claimed. -func (mnr *miner) mapWaitForOpenClaimWindow( +// mapWaitForEarliestCreateClaimHeight is intended to be used as a MapFn. It +// calculates and waits for the earliest block height, allowed by the protocol, +// at which a claim can be created for the given session, then emits the session +// **at that moment**. +func (mnr *miner) mapWaitForEarliestCreateClaimHeight( ctx context.Context, session relayer.SessionTree, ) (_ relayer.SessionTree, skip bool) { @@ -262,9 +263,10 @@ func (mnr *miner) mapWaitForOpenClaimWindow( return session, false } -// waitForEarliestCreateClaimHeight returns the earliest block height at which a -// claim can be created. It is calculated from the session end block height, -// on-chain governance parameters, and randomized input. +// waitForEarliestCreateClaimHeight calculates and waits for the earliest block +// height, allowed by the protocol, at which a claim can be created for a session +// with the given sessionEndHeight. It is calculated relative to sessionEndHeight +// using on-chain governance parameters and randomized input. func (mnr *miner) waitForEarliestCreateClaimHeight( ctx context.Context, sessionEndHeight int64, @@ -273,7 +275,7 @@ func (mnr *miner) waitForEarliestCreateClaimHeight( createClaimWindowStartHeight := sessionEndHeight // TODO_TECHDEBT: query the on-chain governance parameter once available. - // + claimproofparams.GovEarliestClaimSubmissionBlocksOffset + // + claimproofparams.GovCreateClaimWindowStartHeightOffset // we wait for createClaimWindowStartHeight to be received before proceeding since we need its hash // to know where this servicer's claim submission window starts. @@ -330,37 +332,38 @@ func (mnr *miner) newMapClaimSessionFn( } } -// mapWaitForOpenProofWindow maps over the claimedSessions observable, applying -// the mapWaitForOpenProofWindow method to each session. It returns an observable -// of the sessions that are ready to be proven which is notified when the respective -// session is ready to be proven. -func (mnr *miner) mapWaitForOpenProofWindow( +// mapWaitForEarliestSubmitProofHeight is intended to be used as a MapFn. It +// calculates and waits for the earliest block height, allowed by the protocol, +// at which a proof can be submitted for the given session, then emits the session +// **at that moment**. +func (mnr *miner) mapWaitForEarliestSubmitProofHeight( ctx context.Context, session relayer.SessionTree, ) (_ relayer.SessionTree, skip bool) { - mnr.waitForEarliestSubmitProofDistributionHeight( + mnr.waitForEarliestSubmitProofHeight( ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), ) return session, false } -// waitForEarliestSubmitProofDistributionHeight returns the earliest block height -// at which a proof can be submitted. It is calculated from the session claim -// creation block height, on-chain governance parameters, and randomized input. -func (mnr *miner) waitForEarliestSubmitProofDistributionHeight( +// waitForEarliestSubmitProofHeight calculates and waits for the earliest block +// height, allowed by the protocol, at which a proof can be submitted for a session +// which was claimed at createClaimHeight. It is calculated relative to +// createClaimHeight using on-chain governance parameters and randomized input. +func (mnr *miner) waitForEarliestSubmitProofHeight( ctx context.Context, createClaimHeight int64, ) { - earliestSubmitProofBlockHeight := createClaimHeight + submitProofWindowStartHeight := createClaimHeight // TODO_TECHDEBT: query the on-chain governance parameter once available. - // + claimproofparams.GovEarliestProofSubmissionBlocksOffset + // + claimproofparams.GovSubmitProofWindowStartHeightOffset - // we wait for earliestSubmitProofBlockHeight to be received before proceeding since we need its hash - log.Printf("waiting for global earliest proof submission earliestSubmitProofBlock height: %d", earliestSubmitProofBlockHeight) - earliestSubmitProofBlock := mnr.waitForBlock(ctx, earliestSubmitProofBlockHeight) + // we wait for submitProofWindowStartHeight to be received before proceeding since we need its hash + log.Printf("waiting for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) + submitProofWindowStartBlock := mnr.waitForBlock(ctx, submitProofWindowStartHeight) - earliestSubmitProofDistributionHeight := protocol.GetEarliestSubmitProofHeight(earliestSubmitProofBlock) - _ = mnr.waitForBlock(ctx, earliestSubmitProofDistributionHeight) + earliestSubmitProofHeight := protocol.GetEarliestSubmitProofHeight(submitProofWindowStartBlock) + _ = mnr.waitForBlock(ctx, earliestSubmitProofHeight) } // newMapProveSessionFn returns a new MapFn that submits a proof for the given diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go index a84d7dc14..330f0fc34 100644 --- a/pkg/relayer/protocol/block_heights.go +++ b/pkg/relayer/protocol/block_heights.go @@ -19,11 +19,11 @@ func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int6 randomNumber := rand.NewSource(rngSeed).Int63() // TODO_TECHDEBT: query the on-chain governance parameter once available. - // randCreateClaimBlockHeightOffset := randomNumber % (claimproofparams.GovLatestClaimSubmissionBlocksInterval - claimproofparams.GovClaimSubmissionBlocksWindow - 1) + // randCreateClaimHeightOffset := randomNumber % (claimproofparams.GovCreateClaimIntervalBlocks - claimproofparams.GovCreateClaimWindowBlocks - 1) _ = randomNumber - randCreateClaimBlockHeightOffset := int64(0) + randCreateClaimHeightOffset := int64(0) - return createClaimWindowStartBlock.Height() + randCreateClaimBlockHeightOffset + return createClaimWindowStartBlock.Height() + randCreateClaimHeightOffset } // GetEarliestSubmitProofHeight returns the earliest block height at which a proof @@ -37,9 +37,9 @@ func GetEarliestSubmitProofHeight(submitProofWindowStartBlock client.Block) int6 randomNumber := rand.NewSource(rngSeed).Int63() // TODO_TECHDEBT: query the on-chain governance parameter once available. - // randSubmitProofBlockHeightOffset := randomNumber % (claimproofparams.GovLatestProofSubmissionBlocksInterval - claimproofparams.GovProofSubmissionBlocksWindow - 1) + // randSubmitProofHeightOffset := randomNumber % (claimproofparams.GovSubmitProofIntervalBlocks - claimproofparams.GovSubmitProofWindowBlocks - 1) _ = randomNumber - randSubmitProofBlockHeightOffset := int64(0) + randSubmitProofHeightOffset := int64(0) - return submitProofWindowStartBlock.Height() + randSubmitProofBlockHeightOffset + return submitProofWindowStartBlock.Height() + randSubmitProofHeightOffset } From a0ffe9d74df403f8bf8a41f5d6031523542fd79b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 13:53:15 +0100 Subject: [PATCH 025/127] refactor: move claim/proof lifecycle concerns to `relayerSessionsManager`. --- pkg/relayer/interface.go | 29 ++-- pkg/relayer/miner/miner.go | 297 +++------------------------------ pkg/relayer/session/claim.go | 116 +++++++++++++ pkg/relayer/session/proof.go | 111 ++++++++++++ pkg/relayer/session/session.go | 81 ++++++++- pkg/relayer/types.go | 10 ++ 6 files changed, 353 insertions(+), 291 deletions(-) create mode 100644 pkg/relayer/session/claim.go create mode 100644 pkg/relayer/session/proof.go create mode 100644 pkg/relayer/types.go diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 52e14212b..d55630e38 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -23,10 +23,10 @@ import ( // is "requested" (through on-chain mechanisms) and the necessary proof // is submitted on-chain. type Miner interface { - MineRelays( + MinedRelays( ctx context.Context, servedRelays observable.Observable[*servicetypes.Relay], - ) + ) observable.Observable[*MinedRelay] } type MinerOption func(Miner) @@ -84,14 +84,23 @@ type RelayServer interface { // ready to be claimed, and handles the creation and retrieval of SMSTs for a given session. // It also handles the creation and retrieval of SMSTs for a given session. type RelayerSessionsManager interface { - // SessionsToClaim returns an observable that notifies of sessions ready to be claimed. - SessionsToClaim() observable.Observable[SessionTree] - - // EnsureSessionTree returns the SMST (Sparse Merkle State Tree) for a given session header. - // It is used to retrieve the SMST and update it when a Relay has been successfully served. - // If the session is seen for the first time, it creates a new SMST for it before returning it. - // An error is returned if the corresponding KVStore for SMST fails to be created. - EnsureSessionTree(sessionHeader *sessiontypes.SessionHeader) (SessionTree, error) + // RelaysToInclude receives an observable of Relays that should be included + // in their respective session's SMST (tree). + RelaysToInclude(relays observable.Observable[*MinedRelay]) + + // Start iterates over the session trees at the end of each, respective, session. + // The session trees are piped through a series of map operations which progress + // them through the claim/proof lifecycle, broadcasting transactions to the + // network as necessary. + Start(ctx context.Context) + + // Stop unsubscribes all observables from the RelaysToInclude observable which + // will close downstream observables as they drain. + // + // TODO_TECHDEBT: Either add a mechanism to wait for draining to complete + // and/or ensure that the state at each pipeline stage is persisted to disk + // and exit as early as possible. + Stop() } type RelayerSessionsManagerOption func(RelayerSessionsManager) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 6c4369e2f..f83e39fdb 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -20,7 +20,6 @@ import ( "context" "crypto/sha256" "hash" - "log" "cosmossdk.io/depinject" @@ -51,17 +50,9 @@ type miner struct { // Injected dependencies sessionManager relayer.RelayerSessionsManager - supplierClient client.SupplierClient blockClient client.BlockClient } -// minedRelay is a wrapper around a relay that has been serialized and hashed. -type minedRelay struct { - servicetypes.Relay - bytes []byte - hash []byte -} - // NewMiner creates a new miner from the given dependencies and options. It // returns an error if it has not been sufficiently configured or supplied. func NewMiner( @@ -73,7 +64,6 @@ func NewMiner( if err := depinject.Inject( deps, &mnr.sessionManager, - &mnr.supplierClient, &mnr.blockClient, ); err != nil { return nil, err @@ -83,105 +73,36 @@ func NewMiner( opt(mnr) } - if err := mnr.validateConfigAndSetDefaults(); err != nil { + if err := mnr.setDefaults(); err != nil { return nil, err } return mnr, nil } -// MineRelays kicks off relay mining by mapping the servedRelays observable through +// TODO_IN_THIS_COMMIT: revisit comment... +// MinedRelays kicks off relay mining by mapping the servedRelays observable through // a pipeline which hashes the relay, checks if it's above the mining difficulty, // adds it to the session tree, and then maps any errors to a new observable. // It also starts the claim and proof pipelines which are subsequently driven by // mapping over RelayerSessionsManager's SessionsToClaim return observable. // It does not block as map operations run in their own goroutines. -func (mnr *miner) MineRelays( +func (mnr *miner) MinedRelays( ctx context.Context, servedRelays observable.Observable[*servicetypes.Relay], -) { - // sessiontypes.Relay ==> either.Either[minedRelay] +) observable.Observable[*relayer.MinedRelay] { + // Map servedRelays observable to a new observable of an either type, + // populated with the minedRelay or an error, which is notified after the + // relay has been mined or an error has been encountered, respectively. eitherMinedRelays := channel.Map(ctx, servedRelays, mnr.mapMineRelay) + logging.LogErrors(ctx, filter.EitherError(ctx, eitherMinedRelays)) - // either.Either[minedRelay] ==> error - miningErrors := channel.Map(ctx, eitherMinedRelays, mnr.mapAddRelayToSessionTree) - logging.LogErrors(ctx, miningErrors) - - claimedSessions := mnr.createClaims(ctx) - - mnr.submitProofs(ctx, claimedSessions) -} - -// createClaims maps over the RelaySessionsManager's SessionsToClaim return -// observable. For each claim, it calculates and waits for the earliest block -// height at which it is safe to claim and does so. It then maps any errors to -// a new observable which are subsequently logged. It returns an observable of -// the successfully claimed sessions. It does not block as map operations run -// in their own goroutines. -func (mnr *miner) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { - // Map SessionsToClaim observable to a new observable of the same type which - // is notified when the session is eligible to be claimed. - // relayer.SessionTree ==> relayer.SessionTree - sessionsWithOpenClaimWindow := channel.Map( - ctx, mnr.sessionManager.SessionsToClaim(), - mnr.mapWaitForEarliestCreateClaimHeight, - ) - - failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := - channel.NewObservable[relayer.SessionTree]() - - // Map sessionsWithOpenClaimWindow to a new observable of an either type, - // populated with the session or an error, which is notified after the session - // claim has been created or an error has been encountered, respectively. - eitherClaimedSessions := channel.Map( - ctx, sessionsWithOpenClaimWindow, - mnr.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), - ) - - // TODO_TECHDEBT: pass failed create claim sessions to some retry mechanism. - _ = failedCreateClaimSessions - logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessions)) - - // Map eitherClaimedSessions to a new observable of relayer.SessionTree which - // is notified when the corresponding claim creation succeeded. - return filter.EitherSuccess(ctx, eitherClaimedSessions) -} - -// submitProofs maps over the given claimedSessions observable. For each session, -// it calculates and waits for the earliest block height at which it is safe to -// submit a proof and does so. It then maps any errors to a new observable which -// are subsequently logged. It does not block as map operations run in their own -// goroutines. -func (mnr *miner) submitProofs( - ctx context.Context, - claimedSessions observable.Observable[relayer.SessionTree], -) { - // Map claimedSessions to a new observable of the same type which is notified - // when the session is eligible to be proven. - sessionsWithOpenProofWindow := channel.Map( - ctx, claimedSessions, - mnr.mapWaitForEarliestSubmitProofHeight, - ) - - failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := - channel.NewObservable[relayer.SessionTree]() - - // Map sessionsWithOpenProofWindow to a new observable of an either type, - // populated with the session or an error, which is notified after the session - // proof has been submitted or an error has been encountered, respectively. - eitherProvenSessions := channel.Map( - ctx, sessionsWithOpenProofWindow, - mnr.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), - ) - - // TODO_TECHDEBT: pass failed submit proof sessions to some retry mechanism. - _ = failedSubmitProofSessions - logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessions)) + return filter.EitherSuccess(ctx, eitherMinedRelays) } -// validateConfigAndSetDefaults ensures that the miner has been configured with -// a hasher and uses the default hasher if not. -func (mnr *miner) validateConfigAndSetDefaults() error { +// setDefaults ensures that the miner has been configured with a hasher and uses +// the default hasher if not. +func (mnr *miner) setDefaults() error { if mnr.hasher == nil { mnr.hasher = defaultHasher } @@ -190,16 +111,16 @@ func (mnr *miner) validateConfigAndSetDefaults() error { // mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares // its difficulty to the minimum threshold. If the relay difficulty is sifficient, -// it returns an either populated with the minedRelay value. Otherwise, it skips +// it returns an either populated with the MinedRelay value. Otherwise, it skips // the relay. If it encounters an error, it returns an either populated with the // error. func (mnr *miner) mapMineRelay( _ context.Context, relay *servicetypes.Relay, -) (_ either.Either[*minedRelay], skip bool) { +) (_ either.Either[*relayer.MinedRelay], skip bool) { relayBz, err := relay.Marshal() if err != nil { - return either.Error[*minedRelay](err), true + return either.Error[*relayer.MinedRelay](err), true } // Is it correct that we need to hash the key while smst.Update() could do it @@ -209,190 +130,12 @@ func (mnr *miner) mapMineRelay( mnr.hasher.Reset() if !protocol.BytesDifficultyGreaterThan(relayHash, defaultRelayDifficulty) { - return either.Success[*minedRelay](nil), true + return either.Success[*relayer.MinedRelay](nil), true } - return either.Success(&minedRelay{ + return either.Success(&relayer.MinedRelay{ Relay: *relay, - bytes: relayBz, - hash: relayHash, + Bytes: relayBz, + Hash: relayHash, }), false } - -// mapAddRelayToSessionTree is intended to be used as a MapFn. It adds the relay -// to the session tree. If it encounters an error, it returns the error. Otherwise, -// it skips output (only outputs errors). -func (mnr *miner) mapAddRelayToSessionTree( - _ context.Context, - eitherRelay either.Either[*minedRelay], -) (_ error, skip bool) { - // Propagate any upstream errors. - relay, err := eitherRelay.ValueOrError() - if err != nil { - return err, false - } - - // ensure the session tree exists for this relay - sessionHeader := relay.GetReq().GetMeta().GetSessionHeader() - smst, err := mnr.sessionManager.EnsureSessionTree(sessionHeader) - if err != nil { - log.Printf("failed to ensure session tree: %s\n", err) - return err, false - } - - if err := smst.Update(relay.hash, relay.bytes, 1); err != nil { - log.Printf("failed to update smt: %s\n", err) - return err, false - } - - // Skip because this map function only outputs errors. - return nil, true -} - -// mapWaitForEarliestCreateClaimHeight is intended to be used as a MapFn. It -// calculates and waits for the earliest block height, allowed by the protocol, -// at which a claim can be created for the given session, then emits the session -// **at that moment**. -func (mnr *miner) mapWaitForEarliestCreateClaimHeight( - ctx context.Context, - session relayer.SessionTree, -) (_ relayer.SessionTree, skip bool) { - mnr.waitForEarliestCreateClaimHeight( - ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), - ) - return session, false -} - -// waitForEarliestCreateClaimHeight calculates and waits for the earliest block -// height, allowed by the protocol, at which a claim can be created for a session -// with the given sessionEndHeight. It is calculated relative to sessionEndHeight -// using on-chain governance parameters and randomized input. -func (mnr *miner) waitForEarliestCreateClaimHeight( - ctx context.Context, - sessionEndHeight int64, -) { - // TODO_TECHDEBT: refactor this logic to a shared package. - - createClaimWindowStartHeight := sessionEndHeight - // TODO_TECHDEBT: query the on-chain governance parameter once available. - // + claimproofparams.GovCreateClaimWindowStartHeightOffset - - // we wait for createClaimWindowStartHeight to be received before proceeding since we need its hash - // to know where this servicer's claim submission window starts. - log.Printf("waiting for global earliest claim submission createClaimWindowStartBlock height: %d", createClaimWindowStartHeight) - createClaimWindowStartBlock := mnr.waitForBlock(ctx, createClaimWindowStartHeight) - - log.Printf("received earliest claim submission createClaimWindowStartBlock height: %d, use its hash to have a random submission for the servicer", createClaimWindowStartBlock.Height()) - - earliestCreateClaimHeight := - protocol.GetEarliestCreateClaimHeight(createClaimWindowStartBlock) - - log.Printf("earliest claim submission createClaimWindowStartBlock height for this supplier: %d", earliestCreateClaimHeight) - _ = mnr.waitForBlock(ctx, earliestCreateClaimHeight) -} - -// waitForBlock blocks until the block at the given height (or greater) is -// observed as having been committed. -func (mnr *miner) waitForBlock(ctx context.Context, height int64) client.Block { - subscription := mnr.blockClient.CommittedBlocksSequence(ctx).Subscribe(ctx) - defer subscription.Unsubscribe() - - for block := range subscription.Ch() { - if block.Height() >= height { - return block - } - } - - return nil -} - -// newMapClaimSessionFn returns a new MapFn that creates a claim for the given -// session. Any session which encouters an error while creating a claim is sent -// on the failedCreateClaimSessions channel. -func (mnr *miner) newMapClaimSessionFn( - failedCreateClaimSessions chan<- relayer.SessionTree, -) channel.MapFn[relayer.SessionTree, either.SessionTree] { - return func( - ctx context.Context, - session relayer.SessionTree, - ) (_ either.SessionTree, skip bool) { - // this session should no longer be updated - claimRoot, err := session.Flush() - if err != nil { - return either.Error[relayer.SessionTree](err), false - } - - sessionHeader := session.GetSessionHeader() - if err := mnr.supplierClient.CreateClaim(ctx, *sessionHeader, claimRoot); err != nil { - failedCreateClaimSessions <- session - return either.Error[relayer.SessionTree](err), false - } - - return either.Success(session), false - } -} - -// mapWaitForEarliestSubmitProofHeight is intended to be used as a MapFn. It -// calculates and waits for the earliest block height, allowed by the protocol, -// at which a proof can be submitted for the given session, then emits the session -// **at that moment**. -func (mnr *miner) mapWaitForEarliestSubmitProofHeight( - ctx context.Context, - session relayer.SessionTree, -) (_ relayer.SessionTree, skip bool) { - mnr.waitForEarliestSubmitProofHeight( - ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), - ) - return session, false -} - -// waitForEarliestSubmitProofHeight calculates and waits for the earliest block -// height, allowed by the protocol, at which a proof can be submitted for a session -// which was claimed at createClaimHeight. It is calculated relative to -// createClaimHeight using on-chain governance parameters and randomized input. -func (mnr *miner) waitForEarliestSubmitProofHeight( - ctx context.Context, - createClaimHeight int64, -) { - submitProofWindowStartHeight := createClaimHeight - // TODO_TECHDEBT: query the on-chain governance parameter once available. - // + claimproofparams.GovSubmitProofWindowStartHeightOffset - - // we wait for submitProofWindowStartHeight to be received before proceeding since we need its hash - log.Printf("waiting for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) - submitProofWindowStartBlock := mnr.waitForBlock(ctx, submitProofWindowStartHeight) - - earliestSubmitProofHeight := protocol.GetEarliestSubmitProofHeight(submitProofWindowStartBlock) - _ = mnr.waitForBlock(ctx, earliestSubmitProofHeight) -} - -// newMapProveSessionFn returns a new MapFn that submits a proof for the given -// session. Any session which encouters errors while submitting a proof is sent -// on the failedSubmitProofSessions channel. -func (mnr *miner) newMapProveSessionFn( - failedSubmitProofSessions chan<- relayer.SessionTree, -) channel.MapFn[relayer.SessionTree, either.SessionTree] { - return func( - ctx context.Context, - session relayer.SessionTree, - ) (_ either.SessionTree, skip bool) { - latestBlock := mnr.blockClient.LatestBlock(ctx) - proof, err := session.ProveClosest(latestBlock.Hash()) - if err != nil { - return either.Error[relayer.SessionTree](err), false - } - - log.Printf("currentBlock: %d, submitting proof", latestBlock.Height()+1) - // SubmitProof ensures on-chain proof inclusion so we can safely prune the tree. - if err := mnr.supplierClient.SubmitProof( - ctx, - *session.GetSessionHeader(), - proof, - ); err != nil { - failedSubmitProofSessions <- session - return either.Error[relayer.SessionTree](err), false - } - - return either.Success(session), false - } -} diff --git a/pkg/relayer/session/claim.go b/pkg/relayer/session/claim.go new file mode 100644 index 000000000..2fc5e796d --- /dev/null +++ b/pkg/relayer/session/claim.go @@ -0,0 +1,116 @@ +package session + +import ( + "context" + "log" + + "github.com/pokt-network/poktroll/pkg/either" + "github.com/pokt-network/poktroll/pkg/observable" + "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/observable/filter" + "github.com/pokt-network/poktroll/pkg/observable/logging" + "github.com/pokt-network/poktroll/pkg/relayer" + "github.com/pokt-network/poktroll/pkg/relayer/protocol" +) + +// createClaims maps over the sessionsToClaim observable. For each claim, it +// calculates and waits for the earliest block height at which it is safe to +// claim and does so. It then maps any errors to a new observable which are +// subsequently logged. It returns an observable of the successfully claimed +// sessions. It does not block as map operations run in their own goroutines. +func (rs *relayerSessionsManager) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { + // Map SessionsToClaim observable to a new observable of the same type which + // is notified when the session is eligible to be claimed. + // relayer.SessionTree ==> relayer.SessionTree + sessionsWithOpenClaimWindow := channel.Map( + ctx, rs.sessionsToClaim, + rs.mapWaitForEarliestCreateClaimHeight, + ) + + failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := + channel.NewObservable[relayer.SessionTree]() + + // Map sessionsWithOpenClaimWindow to a new observable of an either type, + // populated with the session or an error, which is notified after the session + // claim has been created or an error has been encountered, respectively. + eitherClaimedSessions := channel.Map( + ctx, sessionsWithOpenClaimWindow, + rs.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), + ) + + // TODO_TECHDEBT: pass failed create claim sessions to some retry mechanism. + _ = failedCreateClaimSessions + logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessions)) + + // Map eitherClaimedSessions to a new observable of relayer.SessionTree which + // is notified when the corresponding claim creation succeeded. + return filter.EitherSuccess(ctx, eitherClaimedSessions) +} + +// mapWaitForEarliestCreateClaimHeight is intended to be used as a MapFn. It +// calculates and waits for the earliest block height, allowed by the protocol, +// at which a claim can be created for the given session, then emits the session +// **at that moment**. +func (rs *relayerSessionsManager) mapWaitForEarliestCreateClaimHeight( + ctx context.Context, + session relayer.SessionTree, +) (_ relayer.SessionTree, skip bool) { + rs.waitForEarliestCreateClaimHeight( + ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), + ) + return session, false +} + +// waitForEarliestCreateClaimHeight calculates and waits for the earliest block +// height, allowed by the protocol, at which a claim can be created for a session +// with the given sessionEndHeight. It is calculated relative to sessionEndHeight +// using on-chain governance parameters and randomized input. +func (rs *relayerSessionsManager) waitForEarliestCreateClaimHeight( + ctx context.Context, + sessionEndHeight int64, +) { + // TODO_TECHDEBT: refactor this logic to a shared package. + + createClaimWindowStartHeight := sessionEndHeight + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // + claimproofparams.GovCreateClaimWindowStartHeightOffset + + // we wait for createClaimWindowStartHeight to be received before proceeding since we need its hash + // to know where this servicer's claim submission window starts. + log.Printf("waiting for global earliest claim submission createClaimWindowStartBlock height: %d", createClaimWindowStartHeight) + createClaimWindowStartBlock := rs.waitForBlock(ctx, createClaimWindowStartHeight) + + log.Printf("received earliest claim submission createClaimWindowStartBlock height: %d, use its hash to have a random submission for the servicer", createClaimWindowStartBlock.Height()) + + earliestCreateClaimHeight := + protocol.GetEarliestCreateClaimHeight(createClaimWindowStartBlock) + + log.Printf("earliest claim submission createClaimWindowStartBlock height for this supplier: %d", earliestCreateClaimHeight) + _ = rs.waitForBlock(ctx, earliestCreateClaimHeight) +} + +// newMapClaimSessionFn returns a new MapFn that creates a claim for the given +// session. Any session which encouters an error while creating a claim is sent +// on the failedCreateClaimSessions channel. +func (rs *relayerSessionsManager) newMapClaimSessionFn( + failedCreateClaimSessions chan<- relayer.SessionTree, +) channel.MapFn[relayer.SessionTree, either.SessionTree] { + return func( + ctx context.Context, + session relayer.SessionTree, + ) (_ either.SessionTree, skip bool) { + // this session should no longer be updated + claimRoot, err := session.Flush() + if err != nil { + return either.Error[relayer.SessionTree](err), false + } + + sessionHeader := session.GetSessionHeader() + if err := rs.supplierClient.CreateClaim(ctx, *sessionHeader, claimRoot); err != nil { + failedCreateClaimSessions <- session + return either.Error[relayer.SessionTree](err), false + } + + return either.Success(session), false + } +} diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go new file mode 100644 index 000000000..115f81fd3 --- /dev/null +++ b/pkg/relayer/session/proof.go @@ -0,0 +1,111 @@ +package session + +import ( + "context" + "log" + + "github.com/pokt-network/poktroll/pkg/either" + "github.com/pokt-network/poktroll/pkg/observable" + "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/observable/filter" + "github.com/pokt-network/poktroll/pkg/observable/logging" + "github.com/pokt-network/poktroll/pkg/relayer" + "github.com/pokt-network/poktroll/pkg/relayer/protocol" +) + +// submitProofs maps over the given claimedSessions observable. For each session, +// it calculates and waits for the earliest block height at which it is safe to +// submit a proof and does so. It then maps any errors to a new observable which +// are subsequently logged. It does not block as map operations run in their own +// goroutines. +func (rs *relayerSessionsManager) submitProofs( + ctx context.Context, + claimedSessions observable.Observable[relayer.SessionTree], +) { + // Map claimedSessions to a new observable of the same type which is notified + // when the session is eligible to be proven. + sessionsWithOpenProofWindow := channel.Map( + ctx, claimedSessions, + rs.mapWaitForEarliestSubmitProofHeight, + ) + + failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := + channel.NewObservable[relayer.SessionTree]() + + // Map sessionsWithOpenProofWindow to a new observable of an either type, + // populated with the session or an error, which is notified after the session + // proof has been submitted or an error has been encountered, respectively. + eitherProvenSessions := channel.Map( + ctx, sessionsWithOpenProofWindow, + rs.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), + ) + + // TODO_TECHDEBT: pass failed submit proof sessions to some retry mechanism. + _ = failedSubmitProofSessions + logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessions)) +} + +// mapWaitForEarliestSubmitProofHeight is intended to be used as a MapFn. It +// calculates and waits for the earliest block height, allowed by the protocol, +// at which a proof can be submitted for the given session, then emits the session +// **at that moment**. +func (rs *relayerSessionsManager) mapWaitForEarliestSubmitProofHeight( + ctx context.Context, + session relayer.SessionTree, +) (_ relayer.SessionTree, skip bool) { + rs.waitForEarliestSubmitProofHeight( + ctx, session.GetSessionHeader().GetSessionEndBlockHeight(), + ) + return session, false +} + +// waitForEarliestSubmitProofHeight calculates and waits for the earliest block +// height, allowed by the protocol, at which a proof can be submitted for a session +// which was claimed at createClaimHeight. It is calculated relative to +// createClaimHeight using on-chain governance parameters and randomized input. +func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( + ctx context.Context, + createClaimHeight int64, +) { + submitProofWindowStartHeight := createClaimHeight + // TODO_TECHDEBT: query the on-chain governance parameter once available. + // + claimproofparams.GovSubmitProofWindowStartHeightOffset + + // we wait for submitProofWindowStartHeight to be received before proceeding since we need its hash + log.Printf("waiting for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) + submitProofWindowStartBlock := rs.waitForBlock(ctx, submitProofWindowStartHeight) + + earliestSubmitProofHeight := protocol.GetEarliestSubmitProofHeight(submitProofWindowStartBlock) + _ = rs.waitForBlock(ctx, earliestSubmitProofHeight) +} + +// newMapProveSessionFn returns a new MapFn that submits a proof for the given +// session. Any session which encouters errors while submitting a proof is sent +// on the failedSubmitProofSessions channel. +func (rs *relayerSessionsManager) newMapProveSessionFn( + failedSubmitProofSessions chan<- relayer.SessionTree, +) channel.MapFn[relayer.SessionTree, either.SessionTree] { + return func( + ctx context.Context, + session relayer.SessionTree, + ) (_ either.SessionTree, skip bool) { + latestBlock := rs.blockClient.LatestBlock(ctx) + proof, err := session.ProveClosest(latestBlock.Hash()) + if err != nil { + return either.Error[relayer.SessionTree](err), false + } + + log.Printf("currentBlock: %d, submitting proof", latestBlock.Height()+1) + // SubmitProof ensures on-chain proof inclusion so we can safely prune the tree. + if err := rs.supplierClient.SubmitProof( + ctx, + *session.GetSessionHeader(), + proof, + ); err != nil { + failedSubmitProofSessions <- session + return either.Error[relayer.SessionTree](err), false + } + + return either.Success(session), false + } +} diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 7a45880e1..91c85d7f0 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -6,9 +6,11 @@ import ( "sync" "cosmossdk.io/depinject" + "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/observable/logging" "github.com/pokt-network/poktroll/pkg/relayer" sessiontypes "github.com/pokt-network/poktroll/x/session/types" ) @@ -20,6 +22,8 @@ type sessionsTreesMap = map[int64]map[string]relayer.SessionTree // relayerSessionsManager is an implementation of the RelayerSessions interface. // TODO_TEST: Add tests to the relayerSessionsManager. type relayerSessionsManager struct { + relaysObservable observable.Observable[*relayer.MinedRelay] + // sessionsToClaim notifies about sessions that are ready to be claimed. sessionsToClaim observable.Observable[relayer.SessionTree] @@ -33,6 +37,9 @@ type relayerSessionsManager struct { // blockClient is used to get the notifications of committed blocks. blockClient client.BlockClient + // supplierClient is used to create claims and submit proofs for sessions. + supplierClient client.SupplierClient + // storesDirectory points to a path on disk where KVStore data files are created. storesDirectory string } @@ -50,6 +57,7 @@ func NewRelayerSessions( if err := depinject.Inject( deps, &rs.blockClient, + &rs.supplierClient, ); err != nil { return nil, err } @@ -71,14 +79,40 @@ func NewRelayerSessions( return rs, nil } +// Start iterates over the session trees at the end of each, respective, session. +// The session trees are piped through a series of map operations which progress +// them through the claim/proof lifecycle, broadcasting transactions to the +// network as necessary. +func (rs *relayerSessionsManager) Start(ctx context.Context) { + // Map eitherMinedRelays to a new observable of an error type which is + // notified if an error was encountered while attampting to add the relay to + // the session tree. + miningErrors := channel.Map(ctx, rs.relaysObservable, rs.mapAddRelayToSessionTree) + logging.LogErrors(ctx, miningErrors) + + // Start claim/proof pipeline. + claimedSessionsObservable := rs.createClaims(ctx) + rs.submitProofs(ctx, claimedSessionsObservable) +} + +// Stop unsubscribes all observables from the RelaysToInclude observable which +// will close downstream observables as they drain. +// +// TODO_TECHDEBT: Either add a mechanism to wait for draining to complete +// and/or ensure that the state at each pipeline stage is persisted to disk +// and exit as early as possible. +func (rs *relayerSessionsManager) Stop() { + rs.relaysObservable.UnsubscribeAll() +} + // SessionsToClaim returns an observable that notifies when sessions are ready to be claimed. -func (rs *relayerSessionsManager) SessionsToClaim() observable.Observable[relayer.SessionTree] { - return rs.sessionsToClaim +func (rs *relayerSessionsManager) RelaysToInclude(relays observable.Observable[*relayer.MinedRelay]) { + rs.relaysObservable = relays } -// EnsureSessionTree returns the SessionTree for a given session. +// ensureSessionTree returns the SessionTree for a given session. // If no tree for the session exists, a new SessionTree is created before returning. -func (rs *relayerSessionsManager) EnsureSessionTree(sessionHeader *sessiontypes.SessionHeader) (relayer.SessionTree, error) { +func (rs *relayerSessionsManager) ensureSessionTree(sessionHeader *sessiontypes.SessionHeader) (relayer.SessionTree, error) { rs.sessionsTreesMu.Lock() defer rs.sessionsTreesMu.Unlock() @@ -157,3 +191,42 @@ func (rp *relayerSessionsManager) validateConfig() error { return nil } + +// waitForBlock blocks until the block at the given height (or greater) is +// observed as having been committed. +func (rs *relayerSessionsManager) waitForBlock(ctx context.Context, height int64) client.Block { + subscription := rs.blockClient.CommittedBlocksSequence(ctx).Subscribe(ctx) + defer subscription.Unsubscribe() + + for block := range subscription.Ch() { + if block.Height() >= height { + return block + } + } + + return nil +} + +// mapAddRelayToSessionTree is intended to be used as a MapFn. It adds the relay +// to the session tree. If it encounters an error, it returns the error. Otherwise, +// it skips output (only outputs errors). +func (rs *relayerSessionsManager) mapAddRelayToSessionTree( + _ context.Context, + relay *relayer.MinedRelay, +) (_ error, skip bool) { + // ensure the session tree exists for this relay + sessionHeader := relay.GetReq().GetMeta().GetSessionHeader() + smst, err := rs.ensureSessionTree(sessionHeader) + if err != nil { + log.Printf("failed to ensure session tree: %s\n", err) + return err, false + } + + if err := smst.Update(relay.Hash, relay.Bytes, 1); err != nil { + log.Printf("failed to update smt: %s\n", err) + return err, false + } + + // Skip because this map function only outputs errors. + return nil, true +} diff --git a/pkg/relayer/types.go b/pkg/relayer/types.go new file mode 100644 index 000000000..1216dd25e --- /dev/null +++ b/pkg/relayer/types.go @@ -0,0 +1,10 @@ +package relayer + +import "github.com/pokt-network/poktroll/x/service/types" + +// MinedRelay is a wrapper around a relay that has been serialized and hashed. +type MinedRelay struct { + types.Relay + Bytes []byte + Hash []byte +} From 201963758318058bff8fcbe5b71c71b3286c1a1e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 14:08:35 +0100 Subject: [PATCH 026/127] chore: review feedback improvements --- pkg/relayer/miner/miner.go | 33 ++++++++++++++++----------------- pkg/relayer/session/claim.go | 12 ++++++------ pkg/relayer/session/proof.go | 8 ++++---- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index f83e39fdb..7e1ef820c 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -10,7 +10,7 @@ // is submitted on-chain. // // This is largely accomplished by pipelining observables of relays and sessions -// Through a series of map operations. +// through a series of map operations. // // TODO_TECHDEBT: add architecture diagrams covering observable flows throughout // the miner package. @@ -80,24 +80,20 @@ func NewMiner( return mnr, nil } -// TODO_IN_THIS_COMMIT: revisit comment... -// MinedRelays kicks off relay mining by mapping the servedRelays observable through -// a pipeline which hashes the relay, checks if it's above the mining difficulty, -// adds it to the session tree, and then maps any errors to a new observable. -// It also starts the claim and proof pipelines which are subsequently driven by -// mapping over RelayerSessionsManager's SessionsToClaim return observable. +// MinedRelays maps servedRelaysObs through a pipeline which hashes the relay, +// checks if it's above the mining difficulty, adds it to the session tree if so. // It does not block as map operations run in their own goroutines. func (mnr *miner) MinedRelays( ctx context.Context, - servedRelays observable.Observable[*servicetypes.Relay], + servedRelaysObs observable.Observable[*servicetypes.Relay], ) observable.Observable[*relayer.MinedRelay] { - // Map servedRelays observable to a new observable of an either type, - // populated with the minedRelay or an error, which is notified after the - // relay has been mined or an error has been encountered, respectively. - eitherMinedRelays := channel.Map(ctx, servedRelays, mnr.mapMineRelay) - logging.LogErrors(ctx, filter.EitherError(ctx, eitherMinedRelays)) + // Map servedRelaysObs to a new observable of an either type, populated with + // the minedRelay or an error, which is notified after the relay has been mined + // or an error has been encountered, respectively. + eitherMinedRelaysObs := channel.Map(ctx, servedRelaysObs, mnr.mapMineRelay) + logging.LogErrors(ctx, filter.EitherError(ctx, eitherMinedRelaysObs)) - return filter.EitherSuccess(ctx, eitherMinedRelays) + return filter.EitherSuccess(ctx, eitherMinedRelaysObs) } // setDefaults ensures that the miner has been configured with a hasher and uses @@ -110,7 +106,7 @@ func (mnr *miner) setDefaults() error { } // mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares -// its difficulty to the minimum threshold. If the relay difficulty is sifficient, +// its difficulty to the minimum threshold. If the relay difficulty is sufficient, // it returns an either populated with the MinedRelay value. Otherwise, it skips // the relay. If it encounters an error, it returns an either populated with the // error. @@ -120,10 +116,13 @@ func (mnr *miner) mapMineRelay( ) (_ either.Either[*relayer.MinedRelay], skip bool) { relayBz, err := relay.Marshal() if err != nil { - return either.Error[*relayer.MinedRelay](err), true + return either.Error[*relayer.MinedRelay](err), false } - // Is it correct that we need to hash the key while smst.Update() could do it + // TODO_BLOCKER: Centralize the logic of hashing a relay. It should be live + // alongside signing & verification. + // + // We need to hash the key; it would be nice if smst.Update() could do it // since smst has a reference to the hasher mnr.hasher.Write(relayBz) relayHash := mnr.hasher.Sum(nil) diff --git a/pkg/relayer/session/claim.go b/pkg/relayer/session/claim.go index 2fc5e796d..5ecf2fd42 100644 --- a/pkg/relayer/session/claim.go +++ b/pkg/relayer/session/claim.go @@ -61,10 +61,10 @@ func (rs *relayerSessionsManager) mapWaitForEarliestCreateClaimHeight( return session, false } -// waitForEarliestCreateClaimHeight calculates and waits for the earliest block -// height, allowed by the protocol, at which a claim can be created for a session -// with the given sessionEndHeight. It is calculated relative to sessionEndHeight -// using on-chain governance parameters and randomized input. +// waitForEarliestCreateClaimHeight calculates and waits for (blocking until) the +// earliest block height, allowed by the protocol, at which a claim can be created +// for a session with the given sessionEndHeight. It is calculated relative to +// sessionEndHeight using on-chain governance parameters and randomized input. func (rs *relayerSessionsManager) waitForEarliestCreateClaimHeight( ctx context.Context, sessionEndHeight int64, @@ -93,7 +93,7 @@ func (rs *relayerSessionsManager) waitForEarliestCreateClaimHeight( // session. Any session which encouters an error while creating a claim is sent // on the failedCreateClaimSessions channel. func (rs *relayerSessionsManager) newMapClaimSessionFn( - failedCreateClaimSessions chan<- relayer.SessionTree, + failedCreateClaimSessionsPublishCh chan<- relayer.SessionTree, ) channel.MapFn[relayer.SessionTree, either.SessionTree] { return func( ctx context.Context, @@ -107,7 +107,7 @@ func (rs *relayerSessionsManager) newMapClaimSessionFn( sessionHeader := session.GetSessionHeader() if err := rs.supplierClient.CreateClaim(ctx, *sessionHeader, claimRoot); err != nil { - failedCreateClaimSessions <- session + failedCreateClaimSessionsPublishCh <- session return either.Error[relayer.SessionTree](err), false } diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 115f81fd3..dbead83be 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -59,10 +59,10 @@ func (rs *relayerSessionsManager) mapWaitForEarliestSubmitProofHeight( return session, false } -// waitForEarliestSubmitProofHeight calculates and waits for the earliest block -// height, allowed by the protocol, at which a proof can be submitted for a session -// which was claimed at createClaimHeight. It is calculated relative to -// createClaimHeight using on-chain governance parameters and randomized input. +// waitForEarliestSubmitProofHeight calculates and waits for (blocking until) the +// earliest block height, allowed by the protocol, at which a proof can be submitted +// for a session which was claimed at createClaimHeight. It is calculated relative +// to createClaimHeight using on-chain governance parameters and randomized input. func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( ctx context.Context, createClaimHeight int64, From 709f661cdb819ac4edf9eef1e0c5ee51dbbb6dd2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 14:20:00 +0100 Subject: [PATCH 027/127] chore: review feedback improvements --- pkg/relayer/interface.go | 28 ++++++++++------------------ pkg/relayer/session/session.go | 10 +++++----- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index d55630e38..f822ab2b7 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -12,21 +12,14 @@ import ( sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) -// Miner encapsulates the following responsibilities: -// 1. Mining relays: Served relays are hashed and difficulty is checked. -// Those with sufficient difficulty are added to the session SMST (tree) -// to be applicable for relay volume. -// 2. Creating claims: The session SMST is flushed and an on-chain -// claim is created to the amount of work done by committing -// the tree's root. -// 3. Submitting proofs: A pseudo-random branch from the session SMST -// is "requested" (through on-chain mechanisms) and the necessary proof -// is submitted on-chain. +// Miner is responsible for observing servedRelayObs, hashing and checking the +// difficulty of each, finally publishing those with sufficient difficulty to +// minedRelayObs as they are applicable for relay volume. type Miner interface { MinedRelays( ctx context.Context, - servedRelays observable.Observable[*servicetypes.Relay], - ) observable.Observable[*MinedRelay] + servedRelayObs observable.Observable[*servicetypes.Relay], + ) (minedRelaysObs observable.Observable[*MinedRelay]) } type MinerOption func(Miner) @@ -79,14 +72,13 @@ type RelayServer interface { Service() *sharedtypes.Service } -// RelayerSessionsManager is an interface for managing the relayer's sessions and Sparse -// Merkle Sum Trees (SMSTs). It provides notifications about closing sessions that are -// ready to be claimed, and handles the creation and retrieval of SMSTs for a given session. -// It also handles the creation and retrieval of SMSTs for a given session. +// RelayerSessionsManager is responsible for managing the relayer's session lifecycles. +// It handles the creation and retrieval of SMSTs (trees) for a given session, as +// well as the respective and subsequent claim creation and proof submission. type RelayerSessionsManager interface { - // RelaysToInclude receives an observable of Relays that should be included + // IncludeRelays receives an observable of relays that should be included // in their respective session's SMST (tree). - RelaysToInclude(relays observable.Observable[*MinedRelay]) + IncludeRelays(relayObs observable.Observable[*MinedRelay]) // Start iterates over the session trees at the end of each, respective, session. // The session trees are piped through a series of map operations which progress diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 91c85d7f0..154dd2b50 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -22,7 +22,7 @@ type sessionsTreesMap = map[int64]map[string]relayer.SessionTree // relayerSessionsManager is an implementation of the RelayerSessions interface. // TODO_TEST: Add tests to the relayerSessionsManager. type relayerSessionsManager struct { - relaysObservable observable.Observable[*relayer.MinedRelay] + relayObs observable.Observable[*relayer.MinedRelay] // sessionsToClaim notifies about sessions that are ready to be claimed. sessionsToClaim observable.Observable[relayer.SessionTree] @@ -87,7 +87,7 @@ func (rs *relayerSessionsManager) Start(ctx context.Context) { // Map eitherMinedRelays to a new observable of an error type which is // notified if an error was encountered while attampting to add the relay to // the session tree. - miningErrors := channel.Map(ctx, rs.relaysObservable, rs.mapAddRelayToSessionTree) + miningErrors := channel.Map(ctx, rs.relayObs, rs.mapAddRelayToSessionTree) logging.LogErrors(ctx, miningErrors) // Start claim/proof pipeline. @@ -102,12 +102,12 @@ func (rs *relayerSessionsManager) Start(ctx context.Context) { // and/or ensure that the state at each pipeline stage is persisted to disk // and exit as early as possible. func (rs *relayerSessionsManager) Stop() { - rs.relaysObservable.UnsubscribeAll() + rs.relayObs.UnsubscribeAll() } // SessionsToClaim returns an observable that notifies when sessions are ready to be claimed. -func (rs *relayerSessionsManager) RelaysToInclude(relays observable.Observable[*relayer.MinedRelay]) { - rs.relaysObservable = relays +func (rs *relayerSessionsManager) IncludeRelays(relays observable.Observable[*relayer.MinedRelay]) { + rs.relayObs = relays } // ensureSessionTree returns the SessionTree for a given session. From 394575b7da1d533927e8edff423e725b7c849c0c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 14:25:45 +0100 Subject: [PATCH 028/127] refactor: `miner#hash()` method --- pkg/relayer/miner/miner.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 7e1ef820c..6e750688a 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -35,8 +35,8 @@ import ( ) var ( - _ relayer.Miner = (*miner)(nil) - defaultHasher = sha256.New() + _ relayer.Miner = (*miner)(nil) + defaultHasherConstructor = sha256.New // TODO_BLOCKER: query on-chain governance params once available. // Setting this to 0 to effectively disable mining for now. // I.e., all relays are added to the tree. @@ -45,8 +45,8 @@ var ( // miner implements the relayer.Miner interface. type miner struct { - hasher hash.Hash - relayDifficulty int + hasherConstructor func() hash.Hash + relayDifficulty int // Injected dependencies sessionManager relayer.RelayerSessionsManager @@ -96,11 +96,11 @@ func (mnr *miner) MinedRelays( return filter.EitherSuccess(ctx, eitherMinedRelaysObs) } -// setDefaults ensures that the miner has been configured with a hasher and uses -// the default hasher if not. +// setDefaults ensures that the miner has been configured with a hasherConstructor and uses +// the default hasherConstructor if not. func (mnr *miner) setDefaults() error { - if mnr.hasher == nil { - mnr.hasher = defaultHasher + if mnr.hasherConstructor == nil { + mnr.hasherConstructor = defaultHasherConstructor } return nil } @@ -123,10 +123,8 @@ func (mnr *miner) mapMineRelay( // alongside signing & verification. // // We need to hash the key; it would be nice if smst.Update() could do it - // since smst has a reference to the hasher - mnr.hasher.Write(relayBz) - relayHash := mnr.hasher.Sum(nil) - mnr.hasher.Reset() + // since smst has a reference to the hasherConstructor + relayHash := mnr.hash(relayBz) if !protocol.BytesDifficultyGreaterThan(relayHash, defaultRelayDifficulty) { return either.Success[*relayer.MinedRelay](nil), true @@ -138,3 +136,10 @@ func (mnr *miner) mapMineRelay( Hash: relayHash, }), false } + +// hash constructs a new hasher and hashes the given input bytes. +func (mnr *miner) hash(inputBz []byte) []byte { + hasher := mnr.hasherConstructor() + hasher.Write(inputBz) + return hasher.Sum(nil) +} From 1eae9d2163aaf3f4da7a3f0e11e54b3b23c93bc1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 14:28:28 +0100 Subject: [PATCH 029/127] chore: tidy up --- pkg/relayer/interface.go | 5 +++++ pkg/relayer/miner/miner.go | 20 +++----------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index f822ab2b7..89b1da09c 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -75,6 +75,11 @@ type RelayServer interface { // RelayerSessionsManager is responsible for managing the relayer's session lifecycles. // It handles the creation and retrieval of SMSTs (trees) for a given session, as // well as the respective and subsequent claim creation and proof submission. +// This is largely accomplished by pipelining observables of relays and sessions +// through a series of map operations. +// +// TODO_TECHDEBT: add architecture diagrams covering observable flows throughout +// the relayer package. type RelayerSessionsManager interface { // IncludeRelays receives an observable of relays that should be included // in their respective session's SMST (tree). diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 6e750688a..948076c61 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -1,19 +1,3 @@ -// Package miner encapsulates the responsibilities of the relayer miner interface: -// 1. Mining relays: Served relays are hashed and difficulty is checked. -// Those with sufficient difficulty are added to the session SMST (tree) -// to be applicable for relay volume. -// 2. Creating claims: The session SMST is flushed and an on-chain -// claim is created to the amount of work done by committing -// the tree's root. -// 3. Submitting proofs: A pseudo-random branch from the session SMST -// is "requested" (through on-chain mechanisms) and the necessary proof -// is submitted on-chain. -// -// This is largely accomplished by pipelining observables of relays and sessions -// through a series of map operations. -// -// TODO_TECHDEBT: add architecture diagrams covering observable flows throughout -// the miner package. package miner import ( @@ -43,7 +27,9 @@ var ( defaultRelayDifficulty = 0 ) -// miner implements the relayer.Miner interface. +// Miner is responsible for observing servedRelayObs, hashing and checking the +// difficulty of each, finally publishing those with sufficient difficulty to +// minedRelayObs as they are applicable for relay volume. type miner struct { hasherConstructor func() hash.Hash relayDifficulty int From efb8a4ee5938c686d7bbe813b21a1672f5ba014e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 14:32:43 +0100 Subject: [PATCH 030/127] chore: simplify --- pkg/relayer/miner/miner.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 948076c61..eb2015fbd 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -59,9 +59,7 @@ func NewMiner( opt(mnr) } - if err := mnr.setDefaults(); err != nil { - return nil, err - } + mnr.setDefaults() return mnr, nil } @@ -84,11 +82,10 @@ func (mnr *miner) MinedRelays( // setDefaults ensures that the miner has been configured with a hasherConstructor and uses // the default hasherConstructor if not. -func (mnr *miner) setDefaults() error { +func (mnr *miner) setDefaults() { if mnr.hasherConstructor == nil { mnr.hasherConstructor = defaultHasherConstructor } - return nil } // mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares From b38aa815170ffe277ad44b212459e091c946ee88 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 15:22:32 +0100 Subject: [PATCH 031/127] wip: relayer CLI --- pkg/relayer/cmd/cmd.go | 194 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 pkg/relayer/cmd/cmd.go diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go new file mode 100644 index 000000000..33675adc3 --- /dev/null +++ b/pkg/relayer/cmd/cmd.go @@ -0,0 +1,194 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/signal" + + "cosmossdk.io/depinject" + cosmosclient "github.com/cosmos/cosmos-sdk/client" + cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" + cosmostx "github.com/cosmos/cosmos-sdk/client/tx" + "github.com/spf13/cobra" + + "github.com/pokt-network/poktroll/pkg/client/block" + eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" + "github.com/pokt-network/poktroll/pkg/client/supplier" + "github.com/pokt-network/poktroll/pkg/client/tx" + "github.com/pokt-network/poktroll/pkg/relayer/proxy" +) + +var ( + signingKeyName string + smtStorePath string + sequencerNode string + pocketNode string +) + +func RelayerCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "relayer", + Short: "Run a relayer", + Long: `Run a relayer`, + RunE: runRelayer, + } + + cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") + + // TECHDEBT: integrate these cosmosflags with the client context (i.e. cosmosflags, config, viper, etc.) + // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE). + // Will require more effort than currently justifiable. + cmd.Flags().StringVar(&signingKeyName, "signing-key", "", "Name of the key to sign transactions") + cmd.Flags().StringVar(&smtStorePath, "smt-store", "smt", "Path to the SMT KV store") + // Communication cosmosflags + // TODO_DISCUSS: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. Figure out + // what the defaults should be post alpha. + cmd.Flags().StringVar(&sequencerNode, "sequencer-node", "explicitly omitting default", ": to sequencer/validator node to submit txs") + cmd.Flags().StringVar(&pocketNode, "pocket-node", "explicitly omitting default", ": to full/light pocket node for reading data and listening for on-chain events") + cmd.Flags().String(cosmosflags.FlagNode, "explicitly omitting default", "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") + + return cmd +} + +func runRelayer(cmd *cobra.Command, _ []string) error { + // TODO_IN_THIS_COMMIT: ensure context is always cancelled. + ctx, cancelCtx := context.WithCancel(cmd.Context()) + + // Set --node flag to the --sequencer-node for the client context + cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) + + nodeURL, err := cmd.Flags().GetString(cosmosflags.FlagNode) + if err != nil { + return err + } + + // Construct base dependency injection config. + deps := supplyEventsQueryClient(nodeURL) + deps, err = supplyBlockClient(ctx, deps, nodeURL) + if err != nil { + return err + } + + deps, err = supplyTxClient(ctx, deps, cmd) + if err != nil { + return err + } + + deps, err = supplySupplierClient(deps) + if err != nil { + return err + } + + // -- BEGIN relayer proxy + // INCOMPLETE: this should be populated from some relayer config. + serviceEndpoints := map[string][]string{ + "svc1": {"ws://anvil:8547/"}, + "svc2": {"http://anvil:8547"}, + } + + relayer, err := proxy.NewRelayerProxy( + deps, + relayerProxyOpts, + ) + if err != nil { + cancelCtx() + return err + } + //WithKey(ctx, clientFactory.Keybase(), signingKeyName, address.String(), clientCtx, servicerClient, serviceEndpoints). + //WithServicerClient(servicerClient). + //WithKVStorePath(ctx, filepath.Join(clientCtx.HomeDir, smtStorePath)) + + if err := relayer.Start(ctx); err != nil { + cancelCtx() + return err + } + // -- END relayer proxy + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt) + // Block until we receive an interrupt or kill signal (OS-agnostic) + <-sigCh + + // Signal goroutines to stop + cancelCtx() + // Wait for all goroutines to finish + //wg.Wait() + + // TODO_IN_THIS_COMMIT: synchronize exit + + return nil +} + +func supplyEventsQueryClient(nodeURL string) depinject.Config { + eventsQueryClient := eventsquery.NewEventsQueryClient(nodeURL) + return depinject.Supply(eventsQueryClient) +} + +func supplyBlockClient( + ctx context.Context, + deps depinject.Config, + nodeURL string) (depinject.Config, error) { + blockClient, err := block.NewBlockClient(ctx, deps, nodeURL) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(blockClient)), nil +} + +func supplyTxClient( + ctx context.Context, + deps depinject.Config, + cmd *cobra.Command, +) (depinject.Config, error) { + clientCtx, err := cosmosclient.GetClientTxContext(cmd) + if err != nil { + return nil, err + } + clientFactory, err := cosmostx.NewFactoryCLI(clientCtx, cmd.Flags()) + if err != nil { + return nil, err + } + + deps = depinject.Supply(clientCtx, clientFactory) + txContext, err := tx.NewTxContext(deps) + if err != nil { + return nil, err + } + + deps = depinject.Configs(depinject.Supply(txContext)) + txClient, err := tx.NewTxClient( + ctx, + deps, + tx.WithSigningKeyName(signingKeyName), + // TODO_TECHDEBT: populate this from some config. + tx.WithCommitTimeoutBlocks(tx.DefaultCommitTimeoutHeightOffset), + ) + if err != nil { + return nil, err + } + + return depinject.Configs(depinject.Supply(txClient)), nil +} + +func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { + supplierClient, err := supplier.NewSupplierClient( + deps, + supplier.WithSigningKeyName(signingKeyName), + ) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(supplierClient)), nil +} + +func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { + relayerProxy, err := proxy.NewRelayerProxy(deps) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil +} From c1be2b5928f9a46844dc66548bb5843e359723d6 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 20:00:59 +0100 Subject: [PATCH 032/127] chore: finish first pass --- pkg/relayer/cmd/cmd.go | 121 +++++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 33675adc3..008965902 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "net/url" "os" "os/signal" @@ -16,6 +17,7 @@ import ( eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" "github.com/pokt-network/poktroll/pkg/client/supplier" "github.com/pokt-network/poktroll/pkg/client/tx" + "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/pkg/relayer/proxy" ) @@ -52,58 +54,41 @@ func RelayerCmd() *cobra.Command { } func runRelayer(cmd *cobra.Command, _ []string) error { - // TODO_IN_THIS_COMMIT: ensure context is always cancelled. ctx, cancelCtx := context.WithCancel(cmd.Context()) + // Ensure context cancellation. + defer cancelCtx() - // Set --node flag to the --sequencer-node for the client context - cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) - - nodeURL, err := cmd.Flags().GetString(cosmosflags.FlagNode) - if err != nil { - return err - } - - // Construct base dependency injection config. - deps := supplyEventsQueryClient(nodeURL) - deps, err = supplyBlockClient(ctx, deps, nodeURL) + deps, err := setupRelayerDependencies(ctx, cmd) if err != nil { return err } - deps, err = supplyTxClient(ctx, deps, cmd) - if err != nil { - return err - } - - deps, err = supplySupplierClient(deps) - if err != nil { + var ( + relayerProxy relayer.RelayerProxy + miner relayer.Miner + relayerSessionsManager relayer.RelayerSessionsManager + ) + if err := depinject.Inject( + deps, + &relayerProxy, + &miner, + &relayerSessionsManager, + ); err != nil { return err } - // -- BEGIN relayer proxy - // INCOMPLETE: this should be populated from some relayer config. - serviceEndpoints := map[string][]string{ - "svc1": {"ws://anvil:8547/"}, - "svc2": {"http://anvil:8547"}, - } + // Set up relay pipeline. + servedRelaysObs := relayerProxy.ServedRelays() + minedRelaysObs := miner.MinedRelays(ctx, servedRelaysObs) + relayerSessionsManager.IncludeRelays(minedRelaysObs) - relayer, err := proxy.NewRelayerProxy( - deps, - relayerProxyOpts, - ) - if err != nil { - cancelCtx() - return err - } - //WithKey(ctx, clientFactory.Keybase(), signingKeyName, address.String(), clientCtx, servicerClient, serviceEndpoints). - //WithServicerClient(servicerClient). - //WithKVStorePath(ctx, filepath.Join(clientCtx.HomeDir, smtStorePath)) + // Set up the session (proof/claim) lifecycle pipeline. + relayerSessionsManager.Start(ctx) - if err := relayer.Start(ctx); err != nil { - cancelCtx() + // Start the flow of relays by starting relayer proxy. + if err := relayerProxy.Start(ctx); err != nil { return err } - // -- END relayer proxy sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, os.Interrupt) @@ -120,6 +105,40 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return nil } +func setupRelayerDependencies(ctx context.Context, cmd *cobra.Command) (depinject.Config, error) { + // Set --node flag to the --sequencer-node for the client context + cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) + + nodeURL, err := cmd.Flags().GetString(cosmosflags.FlagNode) + if err != nil { + return nil, err + } + + // Construct base dependency injection config. + deps := supplyEventsQueryClient(nodeURL) + deps, err = supplyBlockClient(ctx, deps, nodeURL) + if err != nil { + return nil, err + } + + deps, err = supplyTxClient(ctx, deps, cmd) + if err != nil { + return nil, err + } + + deps, err = supplySupplierClient(deps) + if err != nil { + return nil, err + } + + deps, err = supplyRelayerProxy(ctx, deps) + if err != nil { + return nil, err + } + + return deps, nil +} + func supplyEventsQueryClient(nodeURL string) depinject.Config { eventsQueryClient := eventsquery.NewEventsQueryClient(nodeURL) return depinject.Supply(eventsQueryClient) @@ -184,11 +203,31 @@ func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(supplierClient)), nil } -func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { - relayerProxy, err := proxy.NewRelayerProxy(deps) +func supplyRelayerProxy( + ctx context.Context, + deps depinject.Config, +) (depinject.Config, error) { + // TODO_INCOMPLETE: this should be populated from some relayerProxy config. + anvilURL, err := url.Parse("ws://anvil:8547/") if err != nil { return nil, err } + proxiedServiceEndpoints := map[string]url.URL{ + "svc1": *anvilURL, + } + + relayerProxy, err := proxy.NewRelayerProxy( + deps, + proxy.WithProxiedServicesEndpoints(proxiedServiceEndpoints), + ) + if err != nil { + return nil, err + } + + if err := relayerProxy.Start(ctx); err != nil { + return nil, err + } + return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil } From 3b2022aa01652de1eb73bad7ef2df424eeb23cf5 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:03:19 +0100 Subject: [PATCH 033/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/protocol/block_heights.go | 4 ++-- pkg/relayer/session/claim.go | 20 +++++++++++--------- pkg/relayer/session/proof.go | 17 +++++++++-------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go index 330f0fc34..b372376f7 100644 --- a/pkg/relayer/protocol/block_heights.go +++ b/pkg/relayer/protocol/block_heights.go @@ -11,7 +11,7 @@ import ( // GetEarliestCreateClaimHeight returns the earliest block height at which a claim // for a session with the given createClaimWindowStartHeight can be created. // -// TODO_TEST(@bryanchriswhite): Add test coverage +// TODO_TEST(@bryanchriswhite): Add test coverage and more logs func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int64 { createClaimWindowStartBlockHash := createClaimWindowStartBlock.Hash() log.Printf("using createClaimWindowStartBlock %d's hash %x as randomness", createClaimWindowStartBlock.Height(), createClaimWindowStartBlockHash) @@ -29,7 +29,7 @@ func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int6 // GetEarliestSubmitProofHeight returns the earliest block height at which a proof // for a session with the given submitProofWindowStartHeight can be submitted. // -// TODO_TEST(@bryanchriswhite): Add test coverage. +// TODO_TEST(@bryanchriswhite): Add test coverage and more logs func GetEarliestSubmitProofHeight(submitProofWindowStartBlock client.Block) int64 { earliestSubmitProofBlockHash := submitProofWindowStartBlock.Hash() log.Printf("using submitProofWindowStartBlock %d's hash %x as randomness", submitProofWindowStartBlock.Height(), earliestSubmitProofBlockHash) diff --git a/pkg/relayer/session/claim.go b/pkg/relayer/session/claim.go index 5ecf2fd42..737ad3d9c 100644 --- a/pkg/relayer/session/claim.go +++ b/pkg/relayer/session/claim.go @@ -13,27 +13,28 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/protocol" ) -// createClaims maps over the sessionsToClaim observable. For each claim, it -// calculates and waits for the earliest block height at which it is safe to -// claim and does so. It then maps any errors to a new observable which are -// subsequently logged. It returns an observable of the successfully claimed -// sessions. It does not block as map operations run in their own goroutines. +// createClaims maps over the sessionsToClaim observable. For each claim, it: +// 1. Calculates the earliest block height at which it is safe to CreateClaim +// 2. Waits for said block and creates the claim on-chain +// 3. Maps errors to a new observable and logs them +// 4. Returns an observable of the successfully claimed sessions +// It DOES NOT BLOCK as map operations run in their own goroutines. func (rs *relayerSessionsManager) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { // Map SessionsToClaim observable to a new observable of the same type which // is notified when the session is eligible to be claimed. // relayer.SessionTree ==> relayer.SessionTree - sessionsWithOpenClaimWindow := channel.Map( + sessionsWithOpenClaimWindowObs := channel.Map( ctx, rs.sessionsToClaim, rs.mapWaitForEarliestCreateClaimHeight, ) - failedCreateClaimSessions, failedCreateClaimSessionsPublishCh := + failedCreateClaimSessionsObs, failedCreateClaimSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() // Map sessionsWithOpenClaimWindow to a new observable of an either type, // populated with the session or an error, which is notified after the session // claim has been created or an error has been encountered, respectively. - eitherClaimedSessions := channel.Map( + eitherClaimedSessionsObs := channel.Map( ctx, sessionsWithOpenClaimWindow, rs.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), ) @@ -65,6 +66,7 @@ func (rs *relayerSessionsManager) mapWaitForEarliestCreateClaimHeight( // earliest block height, allowed by the protocol, at which a claim can be created // for a session with the given sessionEndHeight. It is calculated relative to // sessionEndHeight using on-chain governance parameters and randomized input. +// It IS A BLOCKING function. func (rs *relayerSessionsManager) waitForEarliestCreateClaimHeight( ctx context.Context, sessionEndHeight int64, @@ -77,7 +79,7 @@ func (rs *relayerSessionsManager) waitForEarliestCreateClaimHeight( // we wait for createClaimWindowStartHeight to be received before proceeding since we need its hash // to know where this servicer's claim submission window starts. - log.Printf("waiting for global earliest claim submission createClaimWindowStartBlock height: %d", createClaimWindowStartHeight) + log.Printf("waiting & blocking for global earliest claim submission createClaimWindowStartBlock height: %d", createClaimWindowStartHeight) createClaimWindowStartBlock := rs.waitForBlock(ctx, createClaimWindowStartHeight) log.Printf("received earliest claim submission createClaimWindowStartBlock height: %d, use its hash to have a random submission for the servicer", createClaimWindowStartBlock.Height()) diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index dbead83be..27bd1ea05 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -13,23 +13,24 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/protocol" ) -// submitProofs maps over the given claimedSessions observable. For each session, -// it calculates and waits for the earliest block height at which it is safe to -// submit a proof and does so. It then maps any errors to a new observable which -// are subsequently logged. It does not block as map operations run in their own -// goroutines. +// submitProofs maps over the given claimedSessions observable. +// For each session, it: +// 1. Calculates the earliest block height at which to submit a proof +// 2. Waits for said height and submits the proof on-chain +// 3. Maps errors to a new observable and logs them +// It DOES NOT BLOCKas map operations run in their own goroutines. func (rs *relayerSessionsManager) submitProofs( ctx context.Context, - claimedSessions observable.Observable[relayer.SessionTree], + claimedSessionsObs observable.Observable[relayer.SessionTree], ) { // Map claimedSessions to a new observable of the same type which is notified // when the session is eligible to be proven. - sessionsWithOpenProofWindow := channel.Map( + sessionsWithOpenProofWindowObs := channel.Map( ctx, claimedSessions, rs.mapWaitForEarliestSubmitProofHeight, ) - failedSubmitProofSessions, failedSubmitProveSessionsPublishCh := + failedSubmitProofSessionsObs, failedSubmitProveSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() // Map sessionsWithOpenProofWindow to a new observable of an either type, From 967dddc5a000be61dadf4f5fb8576f16cae984d1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:05:32 +0100 Subject: [PATCH 034/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/miner/miner.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index eb2015fbd..02a645e8f 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -22,7 +22,7 @@ var ( _ relayer.Miner = (*miner)(nil) defaultHasherConstructor = sha256.New // TODO_BLOCKER: query on-chain governance params once available. - // Setting this to 0 to effectively disable mining for now. + // Setting this to 0 to effectively disables mining for now. // I.e., all relays are added to the tree. defaultRelayDifficulty = 0 ) @@ -31,7 +31,7 @@ var ( // difficulty of each, finally publishing those with sufficient difficulty to // minedRelayObs as they are applicable for relay volume. type miner struct { - hasherConstructor func() hash.Hash + relayHasher func() hash.Hash relayDifficulty int // Injected dependencies @@ -64,15 +64,17 @@ func NewMiner( return mnr, nil } -// MinedRelays maps servedRelaysObs through a pipeline which hashes the relay, -// checks if it's above the mining difficulty, adds it to the session tree if so. -// It does not block as map operations run in their own goroutines. +// MinedRelays maps servedRelaysObs through a pipeline which: +// 1. Hashes the relay +// 2. Checks if it's above the mining difficulty +// 3. Adds it to the session tree if so +// It DOES NOT BLOCK as map operations run in their own goroutines. func (mnr *miner) MinedRelays( ctx context.Context, servedRelaysObs observable.Observable[*servicetypes.Relay], ) observable.Observable[*relayer.MinedRelay] { // Map servedRelaysObs to a new observable of an either type, populated with - // the minedRelay or an error, which is notified after the relay has been mined + // the minedRelay or an error. It is notified after the relay has been mined // or an error has been encountered, respectively. eitherMinedRelaysObs := channel.Map(ctx, servedRelaysObs, mnr.mapMineRelay) logging.LogErrors(ctx, filter.EitherError(ctx, eitherMinedRelaysObs)) @@ -88,11 +90,11 @@ func (mnr *miner) setDefaults() { } } -// mapMineRelay is intended to be used as a MapFn. It hashes the relay and compares -// its difficulty to the minimum threshold. If the relay difficulty is sufficient, -// it returns an either populated with the MinedRelay value. Otherwise, it skips -// the relay. If it encounters an error, it returns an either populated with the -// error. +// mapMineRelay is intended to be used as a MapFn. +// 1. It hashes the relay and compares its difficult to the minimum threshold. +// 2. If the relay difficulty is sufficient -> return an Either[MineRelay Value] +// 3. If an error is encountered -> return an Either[error] +// 4. Otherwise, skip the relay. func (mnr *miner) mapMineRelay( _ context.Context, relay *servicetypes.Relay, @@ -102,17 +104,19 @@ func (mnr *miner) mapMineRelay( return either.Error[*relayer.MinedRelay](err), false } - // TODO_BLOCKER: Centralize the logic of hashing a relay. It should be live + // TODO_BLOCKER: Centralize the logic of hashing a relay. It should live // alongside signing & verification. // - // We need to hash the key; it would be nice if smst.Update() could do it + // TODO_IMPROVE: We need to hash the key; it would be nice if smst.Update() could do it // since smst has a reference to the hasherConstructor relayHash := mnr.hash(relayBz) + // The relay IS NOT volume / reward applicable if !protocol.BytesDifficultyGreaterThan(relayHash, defaultRelayDifficulty) { return either.Success[*relayer.MinedRelay](nil), true } + // The relay IS volume / reward applicable return either.Success(&relayer.MinedRelay{ Relay: *relay, Bytes: relayBz, From f9a6fb212fbbd19c2fbac775931023eb4356c17a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:06:52 +0100 Subject: [PATCH 035/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/session/proof.go | 10 +++++++--- pkg/relayer/session/session.go | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 27bd1ea05..644073e7e 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -36,7 +36,7 @@ func (rs *relayerSessionsManager) submitProofs( // Map sessionsWithOpenProofWindow to a new observable of an either type, // populated with the session or an error, which is notified after the session // proof has been submitted or an error has been encountered, respectively. - eitherProvenSessions := channel.Map( + eitherProvenSessionsObs := channel.Map( ctx, sessionsWithOpenProofWindow, rs.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), ) @@ -64,6 +64,7 @@ func (rs *relayerSessionsManager) mapWaitForEarliestSubmitProofHeight( // earliest block height, allowed by the protocol, at which a proof can be submitted // for a session which was claimed at createClaimHeight. It is calculated relative // to createClaimHeight using on-chain governance parameters and randomized input. +// It IS A BLOCKING function. func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( ctx context.Context, createClaimHeight int64, @@ -73,7 +74,7 @@ func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( // + claimproofparams.GovSubmitProofWindowStartHeightOffset // we wait for submitProofWindowStartHeight to be received before proceeding since we need its hash - log.Printf("waiting for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) + log.Printf("waiting and blocking for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) submitProofWindowStartBlock := rs.waitForBlock(ctx, submitProofWindowStartHeight) earliestSubmitProofHeight := protocol.GetEarliestSubmitProofHeight(submitProofWindowStartBlock) @@ -84,12 +85,15 @@ func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( // session. Any session which encouters errors while submitting a proof is sent // on the failedSubmitProofSessions channel. func (rs *relayerSessionsManager) newMapProveSessionFn( - failedSubmitProofSessions chan<- relayer.SessionTree, + failedSubmitProofSessionsCh chan<- relayer.SessionTree, ) channel.MapFn[relayer.SessionTree, either.SessionTree] { return func( ctx context.Context, session relayer.SessionTree, ) (_ either.SessionTree, skip bool) { + // TODO_BLOCKER: The block that'll be used as a source of entropy for which + // branch(es) to prove should be deterministic and use on-chain governance params + // rather than latest. latestBlock := rs.blockClient.LatestBlock(ctx) proof, err := session.ProveClosest(latestBlock.Hash()) if err != nil { diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 154dd2b50..9e1d27aac 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -87,11 +87,11 @@ func (rs *relayerSessionsManager) Start(ctx context.Context) { // Map eitherMinedRelays to a new observable of an error type which is // notified if an error was encountered while attampting to add the relay to // the session tree. - miningErrors := channel.Map(ctx, rs.relayObs, rs.mapAddRelayToSessionTree) + miningErrorsObs := channel.Map(ctx, rs.relayObs, rs.mapAddRelayToSessionTree) logging.LogErrors(ctx, miningErrors) // Start claim/proof pipeline. - claimedSessionsObservable := rs.createClaims(ctx) + claimedSessionsObs := rs.createClaims(ctx) rs.submitProofs(ctx, claimedSessionsObservable) } From 7d23bd49615b27cef01385869f0459d666f42bc9 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:16:23 +0100 Subject: [PATCH 036/127] chore: review feedback improvements --- pkg/relayer/miner/miner.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 02a645e8f..2e1ef3cba 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -7,7 +7,6 @@ import ( "cosmossdk.io/depinject" - "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" @@ -19,8 +18,8 @@ import ( ) var ( - _ relayer.Miner = (*miner)(nil) - defaultHasherConstructor = sha256.New + _ relayer.Miner = (*miner)(nil) + defaultRelayHasher = sha256.New // TODO_BLOCKER: query on-chain governance params once available. // Setting this to 0 to effectively disables mining for now. // I.e., all relays are added to the tree. @@ -31,12 +30,16 @@ var ( // difficulty of each, finally publishing those with sufficient difficulty to // minedRelayObs as they are applicable for relay volume. type miner struct { + // relayHasher is a function which returns a hash.Hash interfact type. It is + // used to hash serialized relays to measure their mining difficulty. relayHasher func() hash.Hash - relayDifficulty int + // relayDifficulty is the minimum difficulty that a relay must have to be + // volume / reward applicable. + relayDifficulty int // Injected dependencies + // sessionManaager facilitates the session (claim/proof) lifecycle. sessionManager relayer.RelayerSessionsManager - blockClient client.BlockClient } // NewMiner creates a new miner from the given dependencies and options. It @@ -50,7 +53,6 @@ func NewMiner( if err := depinject.Inject( deps, &mnr.sessionManager, - &mnr.blockClient, ); err != nil { return nil, err } @@ -85,8 +87,8 @@ func (mnr *miner) MinedRelays( // setDefaults ensures that the miner has been configured with a hasherConstructor and uses // the default hasherConstructor if not. func (mnr *miner) setDefaults() { - if mnr.hasherConstructor == nil { - mnr.hasherConstructor = defaultHasherConstructor + if mnr.relayHasher == nil { + mnr.relayHasher = defaultRelayHasher } } @@ -126,7 +128,7 @@ func (mnr *miner) mapMineRelay( // hash constructs a new hasher and hashes the given input bytes. func (mnr *miner) hash(inputBz []byte) []byte { - hasher := mnr.hasherConstructor() + hasher := mnr.relayHasher() hasher.Write(inputBz) return hasher.Sum(nil) } From 916c0dd29ffa93a8d94c6fed61632b3ad8cde72d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:19:47 +0100 Subject: [PATCH 037/127] chore: review feedback improvements --- pkg/relayer/miner/miner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 2e1ef3cba..19309aea1 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -29,6 +29,8 @@ var ( // Miner is responsible for observing servedRelayObs, hashing and checking the // difficulty of each, finally publishing those with sufficient difficulty to // minedRelayObs as they are applicable for relay volume. +// +// TODO_BLOCKER: The relay hashing and relay difficulty mechanisms & values must come type miner struct { // relayHasher is a function which returns a hash.Hash interfact type. It is // used to hash serialized relays to measure their mining difficulty. From 04a79b24cd2898ec43cceef67d5e3549ba27166b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:26:33 +0000 Subject: [PATCH 038/127] chore: tidy up cmd creation --- cmd/pocketd/cmd/root.go | 6 ++++++ pkg/relayer/cmd/cmd.go | 42 ++++++++++++++++++++--------------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/cmd/pocketd/cmd/root.go b/cmd/pocketd/cmd/root.go index 5b1fb3276..ee949bfee 100644 --- a/cmd/pocketd/cmd/root.go +++ b/cmd/pocketd/cmd/root.go @@ -43,6 +43,7 @@ import ( "github.com/pokt-network/poktroll/app" appparams "github.com/pokt-network/poktroll/app/params" + relayercmd "github.com/pokt-network/poktroll/pkg/relayer/cmd" ) // NewRootCmd creates a new root command for a Cosmos SDK application @@ -141,6 +142,11 @@ func initRootCmd( addModuleInitFlags, ) + // add relayer command + rootCmd.AddCommand( + relayercmd.RelayerCmd(), + ) + // add keybase, auxiliary RPC, query, and tx child commands rootCmd.AddCommand( rpc.StatusCommand(), diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 008965902..965fd97ff 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -82,23 +82,29 @@ func runRelayer(cmd *cobra.Command, _ []string) error { minedRelaysObs := miner.MinedRelays(ctx, servedRelaysObs) relayerSessionsManager.IncludeRelays(minedRelaysObs) + // Handle interrupts in a goroutine. + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt) + + // Block until we receive an interrupt or kill signal (OS-agnostic) + <-sigCh + + // Signal goroutines to stop + cancelCtx() + }() + // Set up the session (proof/claim) lifecycle pipeline. relayerSessionsManager.Start(ctx) // Start the flow of relays by starting relayer proxy. + // This is a blocking call as it waits for the waitgroup to be done. if err := relayerProxy.Start(ctx); err != nil { return err } - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - // Block until we receive an interrupt or kill signal (OS-agnostic) - <-sigCh - - // Signal goroutines to stop - cancelCtx() // Wait for all goroutines to finish - //wg.Wait() + // wg.Wait() // TODO_IN_THIS_COMMIT: synchronize exit @@ -131,7 +137,7 @@ func setupRelayerDependencies(ctx context.Context, cmd *cobra.Command) (depinjec return nil, err } - deps, err = supplyRelayerProxy(ctx, deps) + deps, err = supplyRelayerProxy(deps) if err != nil { return nil, err } @@ -147,7 +153,8 @@ func supplyEventsQueryClient(nodeURL string) depinject.Config { func supplyBlockClient( ctx context.Context, deps depinject.Config, - nodeURL string) (depinject.Config, error) { + nodeURL string, +) (depinject.Config, error) { blockClient, err := block.NewBlockClient(ctx, deps, nodeURL) if err != nil { return nil, err @@ -170,13 +177,13 @@ func supplyTxClient( return nil, err } - deps = depinject.Supply(clientCtx, clientFactory) + deps = depinject.Configs(deps, depinject.Supply(clientCtx, clientFactory)) txContext, err := tx.NewTxContext(deps) if err != nil { return nil, err } - deps = depinject.Configs(depinject.Supply(txContext)) + deps = depinject.Configs(deps, depinject.Supply(txContext)) txClient, err := tx.NewTxClient( ctx, deps, @@ -188,7 +195,7 @@ func supplyTxClient( return nil, err } - return depinject.Configs(depinject.Supply(txClient)), nil + return depinject.Configs(deps, depinject.Supply(txClient)), nil } func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { @@ -203,10 +210,7 @@ func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(supplierClient)), nil } -func supplyRelayerProxy( - ctx context.Context, - deps depinject.Config, -) (depinject.Config, error) { +func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { // TODO_INCOMPLETE: this should be populated from some relayerProxy config. anvilURL, err := url.Parse("ws://anvil:8547/") if err != nil { @@ -225,9 +229,5 @@ func supplyRelayerProxy( return nil, err } - if err := relayerProxy.Start(ctx); err != nil { - return nil, err - } - return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil } From 55c81180263b419f78c266f2afe2e9a3d764cfbf Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:27:32 +0100 Subject: [PATCH 039/127] fix: incomplete refactor --- pkg/relayer/interface.go | 6 +++--- pkg/relayer/session/claim.go | 19 +++++++++---------- pkg/relayer/session/proof.go | 18 +++++++++--------- pkg/relayer/session/session.go | 14 +++++++------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 89b1da09c..134ac8f53 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -81,9 +81,9 @@ type RelayServer interface { // TODO_TECHDEBT: add architecture diagrams covering observable flows throughout // the relayer package. type RelayerSessionsManager interface { - // IncludeRelays receives an observable of relays that should be included + // InsertRelays receives an observable of relays that should be included // in their respective session's SMST (tree). - IncludeRelays(relayObs observable.Observable[*MinedRelay]) + InsertRelays(minedRelaysObs observable.Observable[*MinedRelay]) // Start iterates over the session trees at the end of each, respective, session. // The session trees are piped through a series of map operations which progress @@ -91,7 +91,7 @@ type RelayerSessionsManager interface { // network as necessary. Start(ctx context.Context) - // Stop unsubscribes all observables from the RelaysToInclude observable which + // Stop unsubscribes all observables from the InsertRelays observable which // will close downstream observables as they drain. // // TODO_TECHDEBT: Either add a mechanism to wait for draining to complete diff --git a/pkg/relayer/session/claim.go b/pkg/relayer/session/claim.go index 737ad3d9c..712e7a9e5 100644 --- a/pkg/relayer/session/claim.go +++ b/pkg/relayer/session/claim.go @@ -13,39 +13,38 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/protocol" ) -// createClaims maps over the sessionsToClaim observable. For each claim, it: +// createClaims maps over the sessionsToClaimObs observable. For each claim, it: // 1. Calculates the earliest block height at which it is safe to CreateClaim // 2. Waits for said block and creates the claim on-chain // 3. Maps errors to a new observable and logs them // 4. Returns an observable of the successfully claimed sessions // It DOES NOT BLOCK as map operations run in their own goroutines. func (rs *relayerSessionsManager) createClaims(ctx context.Context) observable.Observable[relayer.SessionTree] { - // Map SessionsToClaim observable to a new observable of the same type which - // is notified when the session is eligible to be claimed. - // relayer.SessionTree ==> relayer.SessionTree + // Map sessionsToClaimObs to a new observable of the same type which is notified + // when the session is eligible to be claimed. sessionsWithOpenClaimWindowObs := channel.Map( - ctx, rs.sessionsToClaim, + ctx, rs.sessionsToClaimObs, rs.mapWaitForEarliestCreateClaimHeight, ) failedCreateClaimSessionsObs, failedCreateClaimSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() - // Map sessionsWithOpenClaimWindow to a new observable of an either type, + // Map sessionsWithOpenClaimWindowObs to a new observable of an either type, // populated with the session or an error, which is notified after the session // claim has been created or an error has been encountered, respectively. eitherClaimedSessionsObs := channel.Map( - ctx, sessionsWithOpenClaimWindow, + ctx, sessionsWithOpenClaimWindowObs, rs.newMapClaimSessionFn(failedCreateClaimSessionsPublishCh), ) // TODO_TECHDEBT: pass failed create claim sessions to some retry mechanism. - _ = failedCreateClaimSessions - logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessions)) + _ = failedCreateClaimSessionsObs + logging.LogErrors(ctx, filter.EitherError(ctx, eitherClaimedSessionsObs)) // Map eitherClaimedSessions to a new observable of relayer.SessionTree which // is notified when the corresponding claim creation succeeded. - return filter.EitherSuccess(ctx, eitherClaimedSessions) + return filter.EitherSuccess(ctx, eitherClaimedSessionsObs) } // mapWaitForEarliestCreateClaimHeight is intended to be used as a MapFn. It diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 644073e7e..4a7c415aa 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -23,27 +23,27 @@ func (rs *relayerSessionsManager) submitProofs( ctx context.Context, claimedSessionsObs observable.Observable[relayer.SessionTree], ) { - // Map claimedSessions to a new observable of the same type which is notified + // Map claimedSessionsObs to a new observable of the same type which is notified // when the session is eligible to be proven. sessionsWithOpenProofWindowObs := channel.Map( - ctx, claimedSessions, + ctx, claimedSessionsObs, rs.mapWaitForEarliestSubmitProofHeight, ) - failedSubmitProofSessionsObs, failedSubmitProveSessionsPublishCh := + failedSubmitProofSessionsObs, failedSubmitProofSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() // Map sessionsWithOpenProofWindow to a new observable of an either type, // populated with the session or an error, which is notified after the session // proof has been submitted or an error has been encountered, respectively. eitherProvenSessionsObs := channel.Map( - ctx, sessionsWithOpenProofWindow, - rs.newMapProveSessionFn(failedSubmitProveSessionsPublishCh), + ctx, sessionsWithOpenProofWindowObs, + rs.newMapProveSessionFn(failedSubmitProofSessionsPublishCh), ) // TODO_TECHDEBT: pass failed submit proof sessions to some retry mechanism. - _ = failedSubmitProofSessions - logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessions)) + _ = failedSubmitProofSessionsObs + logging.LogErrors(ctx, filter.EitherError(ctx, eitherProvenSessionsObs)) } // mapWaitForEarliestSubmitProofHeight is intended to be used as a MapFn. It @@ -91,7 +91,7 @@ func (rs *relayerSessionsManager) newMapProveSessionFn( ctx context.Context, session relayer.SessionTree, ) (_ either.SessionTree, skip bool) { - // TODO_BLOCKER: The block that'll be used as a source of entropy for which + // TODO_BLOCKER: The block that'll be used as a source of entropy for which // branch(es) to prove should be deterministic and use on-chain governance params // rather than latest. latestBlock := rs.blockClient.LatestBlock(ctx) @@ -107,7 +107,7 @@ func (rs *relayerSessionsManager) newMapProveSessionFn( *session.GetSessionHeader(), proof, ); err != nil { - failedSubmitProofSessions <- session + failedSubmitProofSessionsCh <- session return either.Error[relayer.SessionTree](err), false } diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 9e1d27aac..4cf8b0d2c 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -24,8 +24,8 @@ type sessionsTreesMap = map[int64]map[string]relayer.SessionTree type relayerSessionsManager struct { relayObs observable.Observable[*relayer.MinedRelay] - // sessionsToClaim notifies about sessions that are ready to be claimed. - sessionsToClaim observable.Observable[relayer.SessionTree] + // sessionsToClaimObs notifies about sessions that are ready to be claimed. + sessionsToClaimObs observable.Observable[relayer.SessionTree] // sessionTrees is a map of block heights pointing to a map of SessionTrees // indexed by their sessionId. @@ -70,7 +70,7 @@ func NewRelayerSessions( return nil, err } - rs.sessionsToClaim = channel.MapExpand[client.Block, relayer.SessionTree]( + rs.sessionsToClaimObs = channel.MapExpand[client.Block, relayer.SessionTree]( ctx, rs.blockClient.CommittedBlocksSequence(ctx), rs.mapBlockToSessionsToClaim, @@ -88,14 +88,14 @@ func (rs *relayerSessionsManager) Start(ctx context.Context) { // notified if an error was encountered while attampting to add the relay to // the session tree. miningErrorsObs := channel.Map(ctx, rs.relayObs, rs.mapAddRelayToSessionTree) - logging.LogErrors(ctx, miningErrors) + logging.LogErrors(ctx, miningErrorsObs) // Start claim/proof pipeline. claimedSessionsObs := rs.createClaims(ctx) - rs.submitProofs(ctx, claimedSessionsObservable) + rs.submitProofs(ctx, claimedSessionsObs) } -// Stop unsubscribes all observables from the RelaysToInclude observable which +// Stop unsubscribes all observables from the InsertRelays observable which // will close downstream observables as they drain. // // TODO_TECHDEBT: Either add a mechanism to wait for draining to complete @@ -106,7 +106,7 @@ func (rs *relayerSessionsManager) Stop() { } // SessionsToClaim returns an observable that notifies when sessions are ready to be claimed. -func (rs *relayerSessionsManager) IncludeRelays(relays observable.Observable[*relayer.MinedRelay]) { +func (rs *relayerSessionsManager) InsertRelays(relays observable.Observable[*relayer.MinedRelay]) { rs.relayObs = relays } From c1784f5beb6e1cea7ef67bc4d94a88cf67f05012 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:30:19 +0100 Subject: [PATCH 040/127] chore: simplify --- pkg/relayer/miner/miner.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/relayer/miner/miner.go b/pkg/relayer/miner/miner.go index 19309aea1..79d905ac7 100644 --- a/pkg/relayer/miner/miner.go +++ b/pkg/relayer/miner/miner.go @@ -5,8 +5,6 @@ import ( "crypto/sha256" "hash" - "cosmossdk.io/depinject" - "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" @@ -38,27 +36,15 @@ type miner struct { // relayDifficulty is the minimum difficulty that a relay must have to be // volume / reward applicable. relayDifficulty int - - // Injected dependencies - // sessionManaager facilitates the session (claim/proof) lifecycle. - sessionManager relayer.RelayerSessionsManager } // NewMiner creates a new miner from the given dependencies and options. It // returns an error if it has not been sufficiently configured or supplied. func NewMiner( - deps depinject.Config, opts ...relayer.MinerOption, ) (*miner, error) { mnr := &miner{} - if err := depinject.Inject( - deps, - &mnr.sessionManager, - ); err != nil { - return nil, err - } - for _, opt := range opts { opt(mnr) } From 0bfdc8051802cb7915f89875efe82acd3eedb2cd Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:37:20 +0000 Subject: [PATCH 041/127] chore: add log lines --- pkg/relayer/cmd/cmd.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 965fd97ff..7ca9cdcce 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "log" "net/url" "os" "os/signal" @@ -95,10 +96,12 @@ func runRelayer(cmd *cobra.Command, _ []string) error { }() // Set up the session (proof/claim) lifecycle pipeline. + log.Println("INFO: Starting relayer sessions manager...") relayerSessionsManager.Start(ctx) // Start the flow of relays by starting relayer proxy. // This is a blocking call as it waits for the waitgroup to be done. + log.Println("INFO: Starting relayer proxy...") if err := relayerProxy.Start(ctx); err != nil { return err } From 9efde4f64ff5e5d7961d3e623ec09ad57c0c9f26 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:46:58 +0100 Subject: [PATCH 042/127] wip: react to miner, refactor, construct miner, refactor --- pkg/relayer/cmd/cmd.go | 31 ++++++++++++++++++++++++++----- pkg/relayer/proxy/proxy.go | 5 +++-- pkg/relayer/session/session.go | 3 ++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 7ca9cdcce..702d811b9 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -19,6 +19,7 @@ import ( "github.com/pokt-network/poktroll/pkg/client/supplier" "github.com/pokt-network/poktroll/pkg/client/tx" "github.com/pokt-network/poktroll/pkg/relayer" + "github.com/pokt-network/poktroll/pkg/relayer/miner" "github.com/pokt-network/poktroll/pkg/relayer/proxy" ) @@ -81,7 +82,7 @@ func runRelayer(cmd *cobra.Command, _ []string) error { // Set up relay pipeline. servedRelaysObs := relayerProxy.ServedRelays() minedRelaysObs := miner.MinedRelays(ctx, servedRelaysObs) - relayerSessionsManager.IncludeRelays(minedRelaysObs) + relayerSessionsManager.InsertRelays(minedRelaysObs) // Handle interrupts in a goroutine. go func() { @@ -114,7 +115,10 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return nil } -func setupRelayerDependencies(ctx context.Context, cmd *cobra.Command) (depinject.Config, error) { +func setupRelayerDependencies( + ctx context.Context, + cmd *cobra.Command, +) (deps depinject.Config, err error) { // Set --node flag to the --sequencer-node for the client context cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) @@ -123,8 +127,13 @@ func setupRelayerDependencies(ctx context.Context, cmd *cobra.Command) (depinjec return nil, err } + deps, err = supplyMiner(deps) + if err != nil { + return nil, err + } + // Construct base dependency injection config. - deps := supplyEventsQueryClient(nodeURL) + deps = supplyEventsQueryClient(deps, nodeURL) deps, err = supplyBlockClient(ctx, deps, nodeURL) if err != nil { return nil, err @@ -148,9 +157,21 @@ func setupRelayerDependencies(ctx context.Context, cmd *cobra.Command) (depinjec return deps, nil } -func supplyEventsQueryClient(nodeURL string) depinject.Config { +func supplyMiner( + deps depinject.Config, +) (depinject.Config, error) { + mnr, err := miner.NewMiner() + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(mnr)), nil +} + +func supplyEventsQueryClient(deps depinject.Config, nodeURL string) depinject.Config { eventsQueryClient := eventsquery.NewEventsQueryClient(nodeURL) - return depinject.Supply(eventsQueryClient) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)) } func supplyBlockClient( diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index ea73031c8..09b05539b 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -112,8 +112,9 @@ func NewRelayerProxy( return rp, nil } -// Start concurrently starts all advertised relay servers and returns an error if any of them fails to start. -// This method is blocking until all RelayServers are started. +// Start concurrently starts all advertised relay servers and returns an error +// if any of them errors. +// This method is blocking until all RelayServers are stopped. func (rp *relayerProxy) Start(ctx context.Context) error { // The provided services map is built from the supplier's on-chain advertised information, // which is a runtime parameter that can be changed by the supplier. diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 4cf8b0d2c..4af3eaeea 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -79,10 +79,11 @@ func NewRelayerSessions( return rs, nil } -// Start iterates over the session trees at the end of each, respective, session. +// Start maps over the session trees at the end of each, respective, session. // The session trees are piped through a series of map operations which progress // them through the claim/proof lifecycle, broadcasting transactions to the // network as necessary. +// It IS NOT blocking as map operations run in their own goroutines. func (rs *relayerSessionsManager) Start(ctx context.Context) { // Map eitherMinedRelays to a new observable of an error type which is // notified if an error was encountered while attampting to add the relay to From 3de8b3e089e6509e32c6c9e4568645f912d70c41 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 21:48:42 +0100 Subject: [PATCH 043/127] chore: cleanup --- pkg/relayer/cmd/cmd.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 702d811b9..216d53078 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -32,6 +32,7 @@ var ( func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ + // TODO_DISCUSS: do we want to rename this to `relay-miner`? Use: "relayer", Short: "Run a relayer", Long: `Run a relayer`, @@ -107,11 +108,7 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return err } - // Wait for all goroutines to finish - // wg.Wait() - - // TODO_IN_THIS_COMMIT: synchronize exit - + log.Println("INFO: Relayer proxy stopped; exiting") return nil } @@ -132,8 +129,8 @@ func setupRelayerDependencies( return nil, err } - // Construct base dependency injection config. deps = supplyEventsQueryClient(deps, nodeURL) + deps, err = supplyBlockClient(ctx, deps, nodeURL) if err != nil { return nil, err From 059b7dffe94884fbeec864463fadf12b1b2f165d Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 10 Nov 2023 22:26:10 +0100 Subject: [PATCH 044/127] chore: Reflect responsibility changes of session manager --- pkg/relayer/interface.go | 3 +-- pkg/relayer/relayminer.go | 31 +++++++++++++++++++++++++------ pkg/relayer/session/session.go | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 134ac8f53..539361f2d 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -7,7 +7,6 @@ import ( "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/x/service/types" - servicetypes "github.com/pokt-network/poktroll/x/service/types" sessiontypes "github.com/pokt-network/poktroll/x/session/types" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) @@ -18,7 +17,7 @@ import ( type Miner interface { MinedRelays( ctx context.Context, - servedRelayObs observable.Observable[*servicetypes.Relay], + servedRelayObs observable.Observable[*types.Relay], ) (minedRelaysObs observable.Observable[*MinedRelay]) } diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index f4f8b6981..dbbe25aec 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -2,6 +2,7 @@ package relayer import ( "context" + "log" "cosmossdk.io/depinject" ) @@ -9,23 +10,30 @@ import ( // relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). // It starts and stops the RelayerProxy and provide the served relays observable to them miner. type relayMiner struct { - relayerProxy RelayerProxy - miner Miner + relayerProxy RelayerProxy + miner Miner + relayerSessionsManager RelayerSessionsManager } // NewRelayMiner creates a new Relayer instance with the given dependencies. // It injects the dependencies into the Relayer instance and returns it. -func NewRelayMiner(deps depinject.Config) (*relayMiner, error) { +func NewRelayMiner(ctx context.Context, deps depinject.Config) (*relayMiner, error) { rel := &relayMiner{} if err := depinject.Inject( deps, &rel.relayerProxy, &rel.miner, + &rel.relayerSessionsManager, ); err != nil { return nil, err } + // Set up relay pipeline + servedRelaysObs := rel.relayerProxy.ServedRelays() + minedRelaysObs := rel.miner.MinedRelays(ctx, servedRelaysObs) + rel.relayerSessionsManager.InsertRelays(minedRelaysObs) + return rel, nil } @@ -33,9 +41,20 @@ func NewRelayMiner(deps depinject.Config) (*relayMiner, error) { // This method is blocking while the relayer proxy is running and returns when Stop is called // or when the relayer proxy fails to start. func (rel *relayMiner) Start(ctx context.Context) error { - // StartMiningRelays does not block, it only subscribes to the served relays observable. - rel.miner.StartMiningRelays(ctx, rel.relayerProxy.ServedRelays()) - return rel.relayerProxy.Start(ctx) + // relayerSessionsManager.Start does not block. + // Set up the session (proof/claim) lifecycle pipeline. + log.Println("INFO: Starting relayer sessions manager...") + rel.relayerSessionsManager.Start(ctx) + + // Start the flow of relays by starting relayer proxy. + // This is a blocking call as it waits for the waitgroup to be done. + log.Println("INFO: Starting relayer proxy...") + if err := rel.relayerProxy.Start(ctx); err != nil { + return err + } + + log.Println("INFO: Relayer proxy stopped; exiting") + return nil } // Stop stops the relayer proxy which in turn stops all advertised relay servers diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 4cf8b0d2c..4a444dfd0 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -85,7 +85,7 @@ func NewRelayerSessions( // network as necessary. func (rs *relayerSessionsManager) Start(ctx context.Context) { // Map eitherMinedRelays to a new observable of an error type which is - // notified if an error was encountered while attampting to add the relay to + // notified if an error was encountered while attempting to add the relay to // the session tree. miningErrorsObs := channel.Map(ctx, rs.relayObs, rs.mapAddRelayToSessionTree) logging.LogErrors(ctx, miningErrorsObs) From 5f9bd5f7d33cd643f1ed868de1a86db09280e3cb Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 10 Nov 2023 22:47:52 +0100 Subject: [PATCH 045/127] feat: Use relay miner to start --- pkg/relayer/cmd/cmd.go | 44 ++++++++++++++++++--------------------- pkg/relayer/relayminer.go | 12 +++++------ 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 216d53078..886667d5d 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -66,25 +66,14 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return err } - var ( - relayerProxy relayer.RelayerProxy - miner relayer.Miner - relayerSessionsManager relayer.RelayerSessionsManager - ) + var relayMiner relayer.RelayMiner if err := depinject.Inject( deps, - &relayerProxy, - &miner, - &relayerSessionsManager, + &relayMiner, ); err != nil { return err } - // Set up relay pipeline. - servedRelaysObs := relayerProxy.ServedRelays() - minedRelaysObs := miner.MinedRelays(ctx, servedRelaysObs) - relayerSessionsManager.InsertRelays(minedRelaysObs) - // Handle interrupts in a goroutine. go func() { sigCh := make(chan os.Signal, 1) @@ -97,18 +86,11 @@ func runRelayer(cmd *cobra.Command, _ []string) error { cancelCtx() }() - // Set up the session (proof/claim) lifecycle pipeline. - log.Println("INFO: Starting relayer sessions manager...") - relayerSessionsManager.Start(ctx) - - // Start the flow of relays by starting relayer proxy. - // This is a blocking call as it waits for the waitgroup to be done. - log.Println("INFO: Starting relayer proxy...") - if err := relayerProxy.Start(ctx); err != nil { - return err - } + // Start the relay miner + log.Println("INFO: Starting relay miner...") + relayMiner.Start(ctx) - log.Println("INFO: Relayer proxy stopped; exiting") + log.Println("INFO: Relay miner stopped; exiting") return nil } @@ -151,6 +133,11 @@ func setupRelayerDependencies( return nil, err } + deps, err = supplyRelayMiner(ctx, deps) + if err != nil { + return nil, err + } + return deps, nil } @@ -252,3 +239,12 @@ func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil } + +func supplyRelayMiner(ctx context.Context, deps depinject.Config) (depinject.Config, error) { + relayMiner, err := relayer.NewRelayMiner(ctx, deps) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(relayMiner)), nil +} diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index dbbe25aec..e8602a2c4 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -7,9 +7,9 @@ import ( "cosmossdk.io/depinject" ) -// relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). +// RelayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). // It starts and stops the RelayerProxy and provide the served relays observable to them miner. -type relayMiner struct { +type RelayMiner struct { relayerProxy RelayerProxy miner Miner relayerSessionsManager RelayerSessionsManager @@ -17,8 +17,8 @@ type relayMiner struct { // NewRelayMiner creates a new Relayer instance with the given dependencies. // It injects the dependencies into the Relayer instance and returns it. -func NewRelayMiner(ctx context.Context, deps depinject.Config) (*relayMiner, error) { - rel := &relayMiner{} +func NewRelayMiner(ctx context.Context, deps depinject.Config) (*RelayMiner, error) { + rel := &RelayMiner{} if err := depinject.Inject( deps, @@ -40,7 +40,7 @@ func NewRelayMiner(ctx context.Context, deps depinject.Config) (*relayMiner, err // Start provides the miner with the served relays observable and starts the relayer proxy. // This method is blocking while the relayer proxy is running and returns when Stop is called // or when the relayer proxy fails to start. -func (rel *relayMiner) Start(ctx context.Context) error { +func (rel *RelayMiner) Start(ctx context.Context) error { // relayerSessionsManager.Start does not block. // Set up the session (proof/claim) lifecycle pipeline. log.Println("INFO: Starting relayer sessions manager...") @@ -59,6 +59,6 @@ func (rel *relayMiner) Start(ctx context.Context) error { // Stop stops the relayer proxy which in turn stops all advertised relay servers // and unsubscribes the miner from the served relays observable. -func (rel *relayMiner) Stop(ctx context.Context) error { +func (rel *RelayMiner) Stop(ctx context.Context) error { return rel.relayerProxy.Stop(ctx) } From 0c4e3533ac6156f9587f973bb002e0c77523475d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 10 Nov 2023 14:44:06 -0800 Subject: [PATCH 046/127] [WIP] Updating relay.feature to run curl command --- docs/static/openapi.yml | 30 +++++++++++++++-------------- e2e/tests/init_test.go | 4 ++-- e2e/tests/node.go | 42 +++++++++++++++++++++++++++++++++++++---- e2e/tests/relay.feature | 2 +- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/docs/static/openapi.yml b/docs/static/openapi.yml index d82f777d7..f55a851f2 100644 --- a/docs/static/openapi.yml +++ b/docs/static/openapi.yml @@ -46480,7 +46480,7 @@ paths: service: title: >- The Service for which the application is - configured + configured for type: object properties: id: @@ -46660,7 +46660,9 @@ paths: type: object properties: service: - title: The Service for which the application is configured + title: >- + The Service for which the application is configured + for type: object properties: id: @@ -47176,7 +47178,7 @@ paths: service: title: >- The Service for which the application is - configured + configured for type: object properties: id: @@ -47243,7 +47245,7 @@ paths: service: title: >- The Service for which the supplier is - configured + configured for type: object properties: id: @@ -76787,7 +76789,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -76871,7 +76873,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -76965,7 +76967,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77016,7 +77018,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77284,7 +77286,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77348,7 +77350,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77538,7 +77540,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77600,7 +77602,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77814,7 +77816,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77943,7 +77945,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 6bd5aeb27..d6015f0e3 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -242,8 +242,8 @@ func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName st s.Fatalf("session for app %s and service %s does not contain supplier %s", appName, serviceId, supplierName) } -func (s *suite) TheApplicationSendsTheSupplierARelayRequestForService(appName string, supplierName string, requestName string, serviceId string) { - // TODO(#126, @Olshansk): Implement this step +func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName, supplierName, serviceId, requestData string) { + s.pocketd.RunCurl(requestData) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { diff --git a/e2e/tests/node.go b/e2e/tests/node.go index e8ed04dc1..ad2ba8b43 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -44,6 +44,7 @@ type commandResult struct { type PocketClient interface { RunCommand(...string) (*commandResult, error) RunCommandOnHost(string, ...string) (*commandResult, error) + RunCurl(string, ...string) (*commandResult, error) } // Ensure that pocketdBin struct fulfills PocketClient @@ -56,7 +57,7 @@ type pocketdBin struct { // RunCommand runs a command on the local machine using the pocketd binary func (p *pocketdBin) RunCommand(args ...string) (*commandResult, error) { - return p.runCmd(args...) + return p.runPocketCmd(args...) } // RunCommandOnHost runs a command on specified host with the given args @@ -65,11 +66,16 @@ func (p *pocketdBin) RunCommandOnHost(rpcUrl string, args ...string) (*commandRe rpcUrl = defaultRPCURL } args = append(args, "--node", rpcUrl) - return p.runCmd(args...) + return p.runPocketCmd(args...) } -// runCmd is a helper to run a command using the local pocketd binary with the flags provided -func (p *pocketdBin) runCmd(args ...string) (*commandResult, error) { +// RunCurl runs a curl command on the local machine +func (p *pocketdBin) RunCurl(data string, args ...string) (*commandResult, error) { + return p.runCurlPostCmd(data, args...) +} + +// runPocketCmd is a helper to run a command using the local pocketd binary with the flags provided +func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { base := []string{"--home", defaultHome} args = append(base, args...) commandStr := "poktrolld " + strings.Join(args, " ") // Create a string representation of the command @@ -95,3 +101,31 @@ func (p *pocketdBin) runCmd(args ...string) (*commandResult, error) { return r, err } + +// runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided +func (p *pocketdBin) runCurlPostCmd(data string, args ...string) (*commandResult, error) { + base := []string{"-X", "POST", "-H", "Content-Type: application/json", "--data", data} + args = append(base, args...) + commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command + cmd := exec.Command("curl", args...) + + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + + err := cmd.Run() + r := &commandResult{ + Command: commandStr, // Set the command string + Stdout: stdoutBuf.String(), + Stderr: stderrBuf.String(), + Err: err, + } + p.result = r + + if err != nil { + // Include the command executed in the error message for context + err = fmt.Errorf("error running command [%s]: %v, stderr: %s", commandStr, err, stderrBuf.String()) + } + + return r, err +} diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature index 4f071bfa6..629f3c5ec 100644 --- a/e2e/tests/relay.feature +++ b/e2e/tests/relay.feature @@ -5,7 +5,7 @@ Feature: Relay Namespace And the application "app1" is staked for service "anvil" And the supplier "supplier1" is staked for service "anvil" And the session for application "app1" and service "anvil" contains the supplier "supplier1" - When the application "app1" sends the supplier "supplier1" a "getBlock" relay request for service "anvil" + When the application "app1" sends the supplier "supplier1" a request for service "anvil" with data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}" Then the application "app1" receives a successful relay response signed by "supplier1" # TODO_TEST(@Olshansk): From 1dcdbbe8cfb05d9520649f2192ea1635ca8a2598 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Fri, 10 Nov 2023 23:49:58 +0100 Subject: [PATCH 047/127] chore: Improve comment about startig relayer proxy --- pkg/relayer/proxy/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 09b05539b..685e13f86 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -114,7 +114,7 @@ func NewRelayerProxy( // Start concurrently starts all advertised relay servers and returns an error // if any of them errors. -// This method is blocking until all RelayServers are stopped. +// This method IS BLOCKING until all RelayServers are stopped. func (rp *relayerProxy) Start(ctx context.Context) error { // The provided services map is built from the supplier's on-chain advertised information, // which is a runtime parameter that can be changed by the supplier. From da46de743654584b2254f963a7b450e9ae9ad179 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 10 Nov 2023 23:53:38 +0100 Subject: [PATCH 048/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 216d53078..82a549fda 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -53,6 +53,13 @@ func RelayerCmd() *cobra.Command { cmd.Flags().StringVar(&pocketNode, "pocket-node", "explicitly omitting default", ": to full/light pocket node for reading data and listening for on-chain events") cmd.Flags().String(cosmosflags.FlagNode, "explicitly omitting default", "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") + // Set --node flag to the --sequencer-node for the client context + err := cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) + if err != nil { + //return nil, err + panic(err) + } + return cmd } @@ -116,22 +123,23 @@ func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, ) (deps depinject.Config, err error) { - // Set --node flag to the --sequencer-node for the client context - cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) - - nodeURL, err := cmd.Flags().GetString(cosmosflags.FlagNode) + // Initizlize deps to with empty depinject config. + deps, err = supplyMiner(depinject.Configs()) if err != nil { return nil, err } - deps, err = supplyMiner(deps) + rpcQueryURL, err := getPocketNodeWebsocketURL(cmd) if err != nil { return nil, err } - deps = supplyEventsQueryClient(deps, nodeURL) + deps, err = supplyEventsQueryClient(deps, rpcQueryURL) + if err != nil { + return nil, err + } - deps, err = supplyBlockClient(ctx, deps, nodeURL) + deps, err = supplyBlockClient(ctx, deps, rpcQueryURL) if err != nil { return nil, err } @@ -165,10 +173,20 @@ func supplyMiner( return depinject.Configs(deps, depinject.Supply(mnr)), nil } -func supplyEventsQueryClient(deps depinject.Config, nodeURL string) depinject.Config { - eventsQueryClient := eventsquery.NewEventsQueryClient(nodeURL) +func supplyEventsQueryClient(deps depinject.Config, pocketNodeWebsocketURL string) (depinject.Config, error) { + eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketURL) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil +} + +// TODO_IN_THIS_COMMIT: move +func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { + pocketNodeHost, err := cmd.Flags().GetString(cosmosflags.FlagNode) + if err != nil { + return "", err + } - return depinject.Configs(deps, depinject.Supply(eventsQueryClient)) + return fmt.Sprintf("ws://%s/websocket", pocketNodeHost), nil } func supplyBlockClient( From 4f9a74f27dc671ac99a1efcf02992e8a90f323c2 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 10 Nov 2023 14:57:35 -0800 Subject: [PATCH 049/127] Continued implementation but still failing --- Makefile | 2 ++ e2e/tests/init_test.go | 6 +++++- e2e/tests/node.go | 19 ++++++++++++------- e2e/tests/relay.feature | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 1f06b4f68..54d126869 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ POKTROLLD_HOME := ./localnet/poktrolld POCKET_NODE = tcp://127.0.0.1:36657 # The pocket rollup node (full node and sequencer in the localnet context) +RELAYER_NODE = http://anvil:8547 # TODO_IN_THIS_COMMIT: UPDATE THIS URL POCKET_ADDR_PREFIX = pokt #################### @@ -137,6 +138,7 @@ go_imports: check_go_version ## Run goimports on all go files test_e2e: ## Run all E2E tests export POCKET_NODE=$(POCKET_NODE) && \ POKTROLLD_HOME=../../$(POKTROLLD_HOME) && \ + RELAYER_NODE=$(RELAYER_NODE) && \ go test -v ./e2e/tests/... -tags=e2e .PHONY: go_test diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index d6015f0e3..b1bf8128f 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -243,7 +243,11 @@ func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName st } func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName, supplierName, serviceId, requestData string) { - s.pocketd.RunCurl(requestData) + res, err := s.pocketd.RunCurl("", requestData) + if err != nil { + s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) + } + fmt.Println("OLSH", res) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { diff --git a/e2e/tests/node.go b/e2e/tests/node.go index ad2ba8b43..72d1045d9 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -21,6 +21,8 @@ var ( defaultRPCHost = "127.0.0.1" // defaultHome is the default home directory for pocketd defaultHome = os.Getenv("POKTROLLD_HOME") + // defaultRelayerURL used by curl commands to send relay requests + defaultRelayerURL = os.Getenv("RELAYER_NODE") ) func init() { @@ -42,9 +44,9 @@ type commandResult struct { // PocketClient is a single function interface for interacting with a node type PocketClient interface { - RunCommand(...string) (*commandResult, error) - RunCommandOnHost(string, ...string) (*commandResult, error) - RunCurl(string, ...string) (*commandResult, error) + RunCommand(args ...string) (*commandResult, error) + RunCommandOnHost(rpcUrl string, args ...string) (*commandResult, error) + RunCurl(rpcUrl string, data string, args ...string) (*commandResult, error) } // Ensure that pocketdBin struct fulfills PocketClient @@ -70,8 +72,11 @@ func (p *pocketdBin) RunCommandOnHost(rpcUrl string, args ...string) (*commandRe } // RunCurl runs a curl command on the local machine -func (p *pocketdBin) RunCurl(data string, args ...string) (*commandResult, error) { - return p.runCurlPostCmd(data, args...) +func (p *pocketdBin) RunCurl(rpcUrl string, data string, args ...string) (*commandResult, error) { + if rpcUrl == "" { + rpcUrl = defaultRelayerURL + } + return p.runCurlPostCmd(rpcUrl, data, args...) } // runPocketCmd is a helper to run a command using the local pocketd binary with the flags provided @@ -103,8 +108,8 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { } // runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided -func (p *pocketdBin) runCurlPostCmd(data string, args ...string) (*commandResult, error) { - base := []string{"-X", "POST", "-H", "Content-Type: application/json", "--data", data} +func (p *pocketdBin) runCurlPostCmd(rpcUrl string, data string, args ...string) (*commandResult, error) { + base := []string{"-X", "POST", "-H", "Content-Type: application/json", "--data", data, rpcUrl} args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command cmd := exec.Command("curl", args...) diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature index 629f3c5ec..484172092 100644 --- a/e2e/tests/relay.feature +++ b/e2e/tests/relay.feature @@ -5,7 +5,7 @@ Feature: Relay Namespace And the application "app1" is staked for service "anvil" And the supplier "supplier1" is staked for service "anvil" And the session for application "app1" and service "anvil" contains the supplier "supplier1" - When the application "app1" sends the supplier "supplier1" a request for service "anvil" with data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}" + When the application "app1" sends the supplier "supplier1" a request for service "anvil" with data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' Then the application "app1" receives a successful relay response signed by "supplier1" # TODO_TEST(@Olshansk): From cc05bac6729cb10606da416ae2bf45035c3b0c1c Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 10 Nov 2023 15:31:59 -0800 Subject: [PATCH 050/127] Getting an invalid request right now but figuring it out... --- Makefile | 2 +- e2e/tests/init_test.go | 2 +- e2e/tests/node.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 54d126869..6aa690a51 100644 --- a/Makefile +++ b/Makefile @@ -137,8 +137,8 @@ go_imports: check_go_version ## Run goimports on all go files .PHONY: test_e2e test_e2e: ## Run all E2E tests export POCKET_NODE=$(POCKET_NODE) && \ + export RELAYER_NODE=$(RELAYER_NODE) && \ POKTROLLD_HOME=../../$(POKTROLLD_HOME) && \ - RELAYER_NODE=$(RELAYER_NODE) && \ go test -v ./e2e/tests/... -tags=e2e .PHONY: go_test diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index b1bf8128f..2c7d398f3 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -247,7 +247,7 @@ func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } - fmt.Println("OLSH", res) + fmt.Println("OLSH Res", res) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 72d1045d9..d610ef54c 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -109,7 +109,7 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { // runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided func (p *pocketdBin) runCurlPostCmd(rpcUrl string, data string, args ...string) (*commandResult, error) { - base := []string{"-X", "POST", "-H", "Content-Type: application/json", "--data", data, rpcUrl} + base := []string{"-v", "-X", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), rpcUrl} args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command cmd := exec.Command("curl", args...) From 47d8685c9de0331aec6663d448229a05622db5a5 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 01:11:43 +0100 Subject: [PATCH 051/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 66 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 82a549fda..0cc022d01 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -21,6 +21,8 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/pkg/relayer/miner" "github.com/pokt-network/poktroll/pkg/relayer/proxy" + "github.com/pokt-network/poktroll/pkg/relayer/session" + suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" ) var ( @@ -134,31 +136,60 @@ func setupRelayerDependencies( return nil, err } + // Has no dependencies. deps, err = supplyEventsQueryClient(deps, rpcQueryURL) if err != nil { return nil, err } + // Depends on EventsQueryClient. deps, err = supplyBlockClient(ctx, deps, rpcQueryURL) if err != nil { return nil, err } - deps, err = supplyTxClient(ctx, deps, cmd) + // Has no dependencies. + deps, err = supplyClientCtxAndTxFactory(deps, cmd) if err != nil { return nil, err } + var clientCtx cosmosclient.Context + if err := depinject.Inject(deps, &clientCtx); err != nil { + panic(err) + } + supplierQuerier := suppliertypes.NewQueryClient(clientCtx) + supplierQuery := &suppliertypes.QueryGetSupplierRequest{Address: ""} + log.Printf("clientCtx: %+v", clientCtx) + _, err = supplierQuerier.Supplier(ctx, supplierQuery) + if err != nil { + panic(err) + } + + // Depends on clientCtx, txFactory, EventsQueryClient, & BlockClient. + deps, err = supplyTxClient(ctx, deps) + if err != nil { + return nil, err + } + + // Depends on txClient & EventsQueryClient. deps, err = supplySupplierClient(deps) if err != nil { return nil, err } + // Depends on clientCtx & BlockClient. deps, err = supplyRelayerProxy(deps) if err != nil { return nil, err } + // Depends on BlockClient & SupplierClient. + deps, err = supplyRelayerSessionsManager(ctx, deps) + if err != nil { + return nil, err + } + return deps, nil } @@ -181,12 +212,12 @@ func supplyEventsQueryClient(deps depinject.Config, pocketNodeWebsocketURL strin // TODO_IN_THIS_COMMIT: move func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { - pocketNodeHost, err := cmd.Flags().GetString(cosmosflags.FlagNode) + pocketNodeURI, err := cmd.Flags().GetString(cosmosflags.FlagNode) if err != nil { return "", err } - return fmt.Sprintf("ws://%s/websocket", pocketNodeHost), nil + return fmt.Sprintf("ws://%s/websocket", pocketNodeURI), nil } func supplyBlockClient( @@ -202,11 +233,12 @@ func supplyBlockClient( return depinject.Configs(deps, depinject.Supply(blockClient)), nil } -func supplyTxClient( - ctx context.Context, +func supplyClientCtxAndTxFactory( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { + cosmosclient.GetClientQueryContext(cmd) + clientCtx, err := cosmosclient.GetClientTxContext(cmd) if err != nil { return nil, err @@ -216,7 +248,13 @@ func supplyTxClient( return nil, err } - deps = depinject.Configs(deps, depinject.Supply(clientCtx, clientFactory)) + return depinject.Configs(deps, depinject.Supply(clientCtx, clientFactory)), nil +} + +func supplyTxClient( + ctx context.Context, + deps depinject.Config, +) (depinject.Config, error) { txContext, err := tx.NewTxContext(deps) if err != nil { return nil, err @@ -262,6 +300,7 @@ func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { relayerProxy, err := proxy.NewRelayerProxy( deps, + proxy.WithSigningKeyName(signingKeyName), proxy.WithProxiedServicesEndpoints(proxiedServiceEndpoints), ) if err != nil { @@ -270,3 +309,18 @@ func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil } + +func supplyRelayerSessionsManager( + ctx context.Context, + deps depinject.Config, +) (depinject.Config, error) { + relayerSessionsManager, err := session.NewRelayerSessions( + ctx, deps, + session.WithStoresDirectory(smtStorePath), + ) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(relayerSessionsManager)), nil +} From 34e52f7d7a203429b525a64be0a9ffaa59fc3182 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 10 Nov 2023 16:19:45 -0800 Subject: [PATCH 052/127] Added service and switched to AppGate --- Makefile | 4 ++-- e2e/tests/init_test.go | 2 +- e2e/tests/node.go | 16 ++++++++-------- pkg/appgateserver/cmd/cmd.go | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 6aa690a51..104950507 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ POKTROLLD_HOME := ./localnet/poktrolld POCKET_NODE = tcp://127.0.0.1:36657 # The pocket rollup node (full node and sequencer in the localnet context) -RELAYER_NODE = http://anvil:8547 # TODO_IN_THIS_COMMIT: UPDATE THIS URL +APPGATE_SERVER = http://localhost:42069 POCKET_ADDR_PREFIX = pokt #################### @@ -137,7 +137,7 @@ go_imports: check_go_version ## Run goimports on all go files .PHONY: test_e2e test_e2e: ## Run all E2E tests export POCKET_NODE=$(POCKET_NODE) && \ - export RELAYER_NODE=$(RELAYER_NODE) && \ + export APPGATE_SERVER=$(APPGATE_SERVER) && \ POKTROLLD_HOME=../../$(POKTROLLD_HOME) && \ go test -v ./e2e/tests/... -tags=e2e diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 2c7d398f3..6124f4d5a 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -243,7 +243,7 @@ func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName st } func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName, supplierName, serviceId, requestData string) { - res, err := s.pocketd.RunCurl("", requestData) + res, err := s.pocketd.RunCurl("", serviceId, requestData) if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } diff --git a/e2e/tests/node.go b/e2e/tests/node.go index d610ef54c..2aee2eba7 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -21,8 +21,8 @@ var ( defaultRPCHost = "127.0.0.1" // defaultHome is the default home directory for pocketd defaultHome = os.Getenv("POKTROLLD_HOME") - // defaultRelayerURL used by curl commands to send relay requests - defaultRelayerURL = os.Getenv("RELAYER_NODE") + // defaultAppGateServerURL used by curl commands to send relay requests + defaultAppGateServerURL = os.Getenv("APPGATE_SERVER") ) func init() { @@ -46,7 +46,7 @@ type commandResult struct { type PocketClient interface { RunCommand(args ...string) (*commandResult, error) RunCommandOnHost(rpcUrl string, args ...string) (*commandResult, error) - RunCurl(rpcUrl string, data string, args ...string) (*commandResult, error) + RunCurl(rpcUrl, service, data string, args ...string) (*commandResult, error) } // Ensure that pocketdBin struct fulfills PocketClient @@ -72,11 +72,11 @@ func (p *pocketdBin) RunCommandOnHost(rpcUrl string, args ...string) (*commandRe } // RunCurl runs a curl command on the local machine -func (p *pocketdBin) RunCurl(rpcUrl string, data string, args ...string) (*commandResult, error) { +func (p *pocketdBin) RunCurl(rpcUrl, service, data string, args ...string) (*commandResult, error) { if rpcUrl == "" { - rpcUrl = defaultRelayerURL + rpcUrl = defaultAppGateServerURL } - return p.runCurlPostCmd(rpcUrl, data, args...) + return p.runCurlPostCmd(rpcUrl, service, data, args...) } // runPocketCmd is a helper to run a command using the local pocketd binary with the flags provided @@ -108,8 +108,8 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { } // runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided -func (p *pocketdBin) runCurlPostCmd(rpcUrl string, data string, args ...string) (*commandResult, error) { - base := []string{"-v", "-X", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), rpcUrl} +func (p *pocketdBin) runCurlPostCmd(rpcUrl string, service string, data string, args ...string) (*commandResult, error) { + base := []string{"-v", "-X", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), fmt.Sprintf("%s/%s", rpcUrl, service)} args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command cmd := exec.Command("curl", args...) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 21052e270..663562e98 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -142,6 +142,6 @@ func setupAppGateServerDependencies(cmd *cobra.Command, ctx context.Context, com return nil, fmt.Errorf("failed to create block client: %w", err) } - // Return the dependencie config. + // Return the dependencies config. return depinject.Supply(clientCtx, blockClient), nil } From beb07573b5d665699e37982a7b13b8f7046cbd35 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 01:20:52 +0100 Subject: [PATCH 053/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 6c64e97b5..ec18040e2 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -124,6 +124,11 @@ func setupRelayerDependencies( return nil, err } + deps, err = supplyMiner(deps) + if err != nil { + return nil, err + } + // Depends on EventsQueryClient. deps, err = supplyBlockClient(ctx, deps, rpcQueryURL) if err != nil { From ec1910f2ab31d661cda77f12c7fff1d8276bf376 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Sat, 11 Nov 2023 01:32:56 +0100 Subject: [PATCH 054/127] chore: Rename falg variables --- pkg/relayer/cmd/cmd.go | 45 ++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 886667d5d..11e9a7edc 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -24,33 +24,32 @@ import ( ) var ( - signingKeyName string - smtStorePath string - sequencerNode string - pocketNode string + flagSigningKeyName string + flagSmtStorePath string + flagSequencerNode string + flagPocketNode string ) func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ - // TODO_DISCUSS: do we want to rename this to `relay-miner`? - Use: "relayer", - Short: "Run a relayer", - Long: `Run a relayer`, + Use: "relayer-miner", + Short: "Run a relay miner", + Long: `Run a relay miner`, RunE: runRelayer, } cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - // TECHDEBT: integrate these cosmosflags with the client context (i.e. cosmosflags, config, viper, etc.) + // TODO_TECHDEBT: integrate these cosmosflags with the client context (i.e. cosmosflags, config, viper, etc.) // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE). // Will require more effort than currently justifiable. - cmd.Flags().StringVar(&signingKeyName, "signing-key", "", "Name of the key to sign transactions") - cmd.Flags().StringVar(&smtStorePath, "smt-store", "smt", "Path to the SMT KV store") + cmd.Flags().StringVar(&flagSigningKeyName, "signing-key", "", "Name of the key to sign transactions") + cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to the SMT KV store") // Communication cosmosflags - // TODO_DISCUSS: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. Figure out - // what the defaults should be post alpha. - cmd.Flags().StringVar(&sequencerNode, "sequencer-node", "explicitly omitting default", ": to sequencer/validator node to submit txs") - cmd.Flags().StringVar(&pocketNode, "pocket-node", "explicitly omitting default", ": to full/light pocket node for reading data and listening for on-chain events") + // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. Figure out + // Figure out what good defaults should be post alpha. + cmd.Flags().StringVar(&flagSequencerNode, "sequencer-node", "explicitly omitting default", ": to sequencer node to submit txs") + cmd.Flags().StringVar(&flagPocketNode, "pocket-node", "explicitly omitting default", ": to full pocket node for reading data and listening for on-chain events") cmd.Flags().String(cosmosflags.FlagNode, "explicitly omitting default", "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") return cmd @@ -61,6 +60,8 @@ func runRelayer(cmd *cobra.Command, _ []string) error { // Ensure context cancellation. defer cancelCtx() + // Sets up the following dependencies: + // Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, RelayMiner dependencies. deps, err := setupRelayerDependencies(ctx, cmd) if err != nil { return err @@ -94,12 +95,14 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return nil } +// setupRelayerDependencies sets up all the dependencies the relay miner needs to run: +// Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, RelayMiner func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, ) (deps depinject.Config, err error) { - // Set --node flag to the --sequencer-node for the client context - cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", sequencerNode)) + // TODO_TECHDEBT: Revisit this decision and see if it can be removed + cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", flagSequencerNode)) nodeURL, err := cmd.Flags().GetString(cosmosflags.FlagNode) if err != nil { @@ -195,7 +198,7 @@ func supplyTxClient( txClient, err := tx.NewTxClient( ctx, deps, - tx.WithSigningKeyName(signingKeyName), + tx.WithSigningKeyName(flagSigningKeyName), // TODO_TECHDEBT: populate this from some config. tx.WithCommitTimeoutBlocks(tx.DefaultCommitTimeoutHeightOffset), ) @@ -209,7 +212,7 @@ func supplyTxClient( func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { supplierClient, err := supplier.NewSupplierClient( deps, - supplier.WithSigningKeyName(signingKeyName), + supplier.WithSigningKeyName(flagSigningKeyName), ) if err != nil { return nil, err @@ -219,14 +222,14 @@ func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { } func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { - // TODO_INCOMPLETE: this should be populated from some relayerProxy config. + // TODO_TECHDEBT: this should be populated from some relayerProxy config. anvilURL, err := url.Parse("ws://anvil:8547/") if err != nil { return nil, err } proxiedServiceEndpoints := map[string]url.URL{ - "svc1": *anvilURL, + "anvil": *anvilURL, } relayerProxy, err := proxy.NewRelayerProxy( From ded793f1630d4c65929d886d222b6d9026387736 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 01:34:56 +0100 Subject: [PATCH 055/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index ec18040e2..ff3e1c4bd 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -124,11 +124,6 @@ func setupRelayerDependencies( return nil, err } - deps, err = supplyMiner(deps) - if err != nil { - return nil, err - } - // Depends on EventsQueryClient. deps, err = supplyBlockClient(ctx, deps, rpcQueryURL) if err != nil { @@ -141,12 +136,18 @@ func setupRelayerDependencies( return nil, err } - var clientCtx cosmosclient.Context - if err := depinject.Inject(deps, &clientCtx); err != nil { + //var clientCtx cosmosclient.Context + //if err := depinject.Inject(deps, &clientCtx); err != nil { + // panic(err) + //} + + clientCtx, err := cosmosclient.GetClientQueryContext(cmd) + if err != nil { panic(err) } supplierQuerier := suppliertypes.NewQueryClient(clientCtx) supplierQuery := &suppliertypes.QueryGetSupplierRequest{Address: ""} + log.Printf("clientCtx: %+v", clientCtx) _, err = supplierQuerier.Supplier(ctx, supplierQuery) if err != nil { @@ -224,8 +225,6 @@ func supplyClientCtxAndTxFactory( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - cosmosclient.GetClientQueryContext(cmd) - clientCtx, err := cosmosclient.GetClientTxContext(cmd) if err != nil { return nil, err From bf936e6b239903c9df267bc6b94f262f12216928 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 01:38:34 +0100 Subject: [PATCH 056/127] revertme: disable tilt relayer service --- Tiltfile | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Tiltfile b/Tiltfile index 1333a406e..bc470c037 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,7 +2,7 @@ load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_resource", "helm_repo") # A list of directories where changes trigger a hot-reload of the sequencer -hot_reload_dirs = ["app", "cmd", "tools", "x"] +hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg/relayer"] # Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values localnet_config_path = "localnet_config.yaml" @@ -11,7 +11,7 @@ localnet_config_defaults = { "gateways": {"count": 1}, # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development - "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, + "helm_chart_local_repo": {"enabled": True, "path": "../helm-charts"}, } localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) localnet_config = {} @@ -122,16 +122,16 @@ helm_resource( image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], ) -helm_resource( - "relayers", - poktroll_chart, - flags=[ - "--values=./localnet/kubernetes/values-common.yaml", - "--set=replicaCount=" + str(localnet_config["relayers"]["count"]), - ], - image_deps=["poktrolld"], - image_keys=[("image.repository", "image.tag")], -) +# helm_resource( +# "relayers", +# poktroll_chart, +# flags=[ +# "--values=./localnet/kubernetes/values-common.yaml", +# "--set=replicaCount=" + str(localnet_config["relayers"]["count"]), +# ], +# image_deps=["poktrolld"], +# image_keys=[("image.repository", "image.tag")], +# ) # Configure tilt resources (tilt labels and port forwards) for all of the nodes above k8s_resource( @@ -145,10 +145,10 @@ k8s_resource( resource_deps=["celestia-rollkit"], port_forwards=["36657", "40004"], ) -k8s_resource( - "relayers", - labels=["blockchains"], - resource_deps=["sequencer"], - port_forwards=["8545", "8546", "40005"], -) +# k8s_resource( +# "relayers", +# labels=["blockchains"], +# resource_deps=["sequencer"], +# port_forwards=["8545", "8546", "40005"], +# ) k8s_resource("anvil", labels=["blockchains"], port_forwards=["8547"]) From 0d9edb9dff4dc403061ba3706ce1959dd5dee460 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sat, 11 Nov 2023 00:38:50 +0000 Subject: [PATCH 057/127] chore: use arg not flag --- pkg/appgateserver/cmd/cmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 21052e270..a9ff2908f 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -132,12 +132,12 @@ func setupAppGateServerDependencies(cmd *cobra.Command, ctx context.Context, com clientCtx := cosmosclient.GetClientContextFromCmd(cmd) // Create the events client. - eventsQueryClient := eventsquery.NewEventsQueryClient(flagCometWebsocketUrl) + eventsQueryClient := eventsquery.NewEventsQueryClient(cometWebsocketUrl) // Create the block client. - log.Printf("INFO: Creating block client, using comet websocket URL: %s...", flagCometWebsocketUrl) + log.Printf("INFO: Creating block client, using comet websocket URL: %s...", cometWebsocketUrl) deps := depinject.Supply(eventsQueryClient) - blockClient, err := blockclient.NewBlockClient(ctx, deps, flagCometWebsocketUrl) + blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) if err != nil { return nil, fmt.Errorf("failed to create block client: %w", err) } From 5c5eace0e445db7808a3ad748f32b5a12c2f4242 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Sat, 11 Nov 2023 01:47:55 +0100 Subject: [PATCH 058/127] chore: rename command --- pkg/relayer/cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 94a1c1a6d..f763184fb 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -34,7 +34,7 @@ var ( func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "relayer-miner", + Use: "relayerminer", Short: "Run a relay miner", Long: `Run a relay miner`, RunE: runRelayer, From ee397d3361d2b1f49d048a8417e32d6d449631fe Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 10 Nov 2023 17:02:23 -0800 Subject: [PATCH 059/127] Debugging checkpoint --- config.yml | 2 +- e2e/tests/init_test.go | 2 +- e2e/tests/relay.feature | 6 ++++++ pkg/appgateserver/jsonrpc.go | 11 ++++++++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/config.yml b/config.yml index f21fe2be6..f6e3c1e89 100644 --- a/config.yml +++ b/config.yml @@ -100,7 +100,7 @@ genesis: - endpoints: - configs: [] rpc_type: JSON_RPC - url: http://anvil:8547 + url: http://localhost:8545 service: id: anvil name: "" diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 6124f4d5a..4532596d9 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -247,7 +247,7 @@ func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } - fmt.Println("OLSH Res", res) + fmt.Println("OLSH Res", res.Stdout) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature index 484172092..f5b6af236 100644 --- a/e2e/tests/relay.feature +++ b/e2e/tests/relay.feature @@ -2,6 +2,12 @@ Feature: Relay Namespace Scenario: App can send relay to Supplier Given the user has the pocketd binary installed + + # TEMP: DELETE + When the user sends "1000" uPOKT from account "app2" to account "app1" + When the user sends "1000" uPOKT from account "app2" to account "supplier1" + # TEMP: DELETE + And the application "app1" is staked for service "anvil" And the supplier "supplier1" is staked for service "anvil" And the session for application "app1" and service "anvil" contains the supplier "supplier1" diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 3be01cca7..0e3f9fe16 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -3,6 +3,7 @@ package appgateserver import ( "bytes" "context" + "fmt" "io" "log" "net/http" @@ -78,6 +79,7 @@ func (app *appGateServer) handleJSONRPCRelay( return err } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) + // relayRequestReader := io.NopCloser(bytes.NewReader(payloadBz)) // Create the HTTP request to send the request to the relayer. relayHTTPRequest := &http.Request{ @@ -87,8 +89,15 @@ func (app *appGateServer) handleJSONRPCRelay( Body: relayRequestReader, } + // TODO: relayminer is currently named relayers + // application (localhost) + // -> appgate (localhost:42069); configured by the gateway/application **off-chain** + // -> relayminer (supplierURL); advertised **on-chain** + // -> anvil (localhost:8547); configured **behind-the-scenes**; chains.json (v0); currently hard-coded + // Perform the HTTP request to the relayer. - log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) + log.Printf("DEBUG: Sending raw payload to signed relay request to %s", supplierUrl) + fmt.Printf("\n~~~~ OLSH %+v \n~~~~\n", relayHTTPRequest) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err From a3a3635593a057ebcbddc32657a8e7771e2a5712 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 02:03:23 +0100 Subject: [PATCH 060/127] wip: debugging - improvments --- pkg/client/tx/context.go | 9 ++++++++- pkg/relayer/cmd/cmd.go | 22 +++++++++++++++++----- pkg/relayer/interface.go | 6 ++++++ pkg/relayer/proxy/proxy.go | 8 +++++++- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/pkg/client/tx/context.go b/pkg/client/tx/context.go index eca32f943..639e26e18 100644 --- a/pkg/client/tx/context.go +++ b/pkg/client/tx/context.go @@ -12,6 +12,7 @@ import ( authclient "github.com/cosmos/cosmos-sdk/x/auth/client" "github.com/pokt-network/poktroll/pkg/client" + "github.com/pokt-network/poktroll/pkg/relayer" ) var _ client.TxContext = (*cosmosTxContext)(nil) @@ -33,14 +34,20 @@ type cosmosTxContext struct { func NewTxContext(deps depinject.Config) (client.TxContext, error) { txCtx := cosmosTxContext{} + var txClientCtx relayer.TxClientContext + if err := depinject.Inject( deps, - &txCtx.clientCtx, + &txClientCtx, &txCtx.txFactory, ); err != nil { return nil, err } + // TODO_CONSIDERATION: it might improve readability to use + // QueryClientContext on the struct. + txCtx.clientCtx = cosmosclient.Context(txClientCtx) + return txCtx, nil } diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index ff3e1c4bd..66aa1a8bc 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -131,7 +131,7 @@ func setupRelayerDependencies( } // Has no dependencies. - deps, err = supplyClientCtxAndTxFactory(deps, cmd) + deps, err = supplyTxClientCtxAndTxFactory(deps, cmd) if err != nil { return nil, err } @@ -167,7 +167,7 @@ func setupRelayerDependencies( } // Depends on clientCtx & BlockClient. - deps, err = supplyRelayerProxy(deps) + deps, err = supplyRelayerProxy(deps, cmd) if err != nil { return nil, err } @@ -221,7 +221,7 @@ func supplyBlockClient( return depinject.Configs(deps, depinject.Supply(blockClient)), nil } -func supplyClientCtxAndTxFactory( +func supplyTxClientCtxAndTxFactory( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { @@ -234,7 +234,8 @@ func supplyClientCtxAndTxFactory( return nil, err } - return depinject.Configs(deps, depinject.Supply(clientCtx, clientFactory)), nil + txClientCtx := relayer.TxClientContext(clientCtx) + return depinject.Configs(deps, depinject.Supply(txClientCtx, clientFactory)), nil } func supplyTxClient( @@ -273,7 +274,10 @@ func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(supplierClient)), nil } -func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { +func supplyRelayerProxy( + deps depinject.Config, + cmd *cobra.Command, +) (depinject.Config, error) { // TODO_INCOMPLETE: this should be populated from some relayerProxy config. anvilURL, err := url.Parse("ws://anvil:8547/") if err != nil { @@ -284,6 +288,14 @@ func supplyRelayerProxy(deps depinject.Config) (depinject.Config, error) { "svc1": *anvilURL, } + clientCtx, err := cosmosclient.GetClientQueryContext(cmd) + if err != nil { + return nil, err + } + + queryClientCtx := relayer.QueryClientContext(clientCtx) + deps = depinject.Configs(deps, depinject.Supply(queryClientCtx)) + relayerProxy, err := proxy.NewRelayerProxy( deps, proxy.WithSigningKeyName(signingKeyName), diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 539361f2d..3153ac9b7 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -3,6 +3,7 @@ package relayer import ( "context" + "github.com/cosmos/cosmos-sdk/client" "github.com/pokt-network/smt" "github.com/pokt-network/poktroll/pkg/observable" @@ -130,3 +131,8 @@ type SessionTree interface { // submitted on-chain and the servicer has confirmed that it has been rewarded. Delete() error } + +// TODO_IN_THIS_COMMIT: comment.. these exist to differentiate in +// depinject; therefore, can't be aliases. +type TxClientContext client.Context +type QueryClientContext client.Context diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 5eee4cd2c..d59ea66a7 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -97,14 +97,20 @@ func NewRelayerProxy( ) (relayer.RelayerProxy, error) { rp := &relayerProxy{} + var queryClientCtx relayer.QueryClientContext + if err := depinject.Inject( deps, - &rp.clientCtx, + &queryClientCtx, &rp.blockClient, ); err != nil { return nil, err } + // TODO_CONSIDERATION: it might improve readability to use + // QueryClientContext on the struct. + rp.clientCtx = sdkclient.Context(queryClientCtx) + servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() rp.servedRelays = servedRelays From 71052b7582c079af2a5e9eae8846254d59e3112e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 14:23:02 +0100 Subject: [PATCH 061/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 848cefa4e..6de57e397 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -22,7 +22,6 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/miner" "github.com/pokt-network/poktroll/pkg/relayer/proxy" "github.com/pokt-network/poktroll/pkg/relayer/session" - suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" ) var ( @@ -76,11 +75,8 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return err } - var relayMiner relayer.RelayMiner - if err := depinject.Inject( - deps, - &relayMiner, - ); err != nil { + relayMiner, err := relayer.NewRelayMiner(ctx, deps) + if err != nil { return err } @@ -139,24 +135,6 @@ func setupRelayerDependencies( return nil, err } - //var clientCtx cosmosclient.Context - //if err := depinject.Inject(deps, &clientCtx); err != nil { - // panic(err) - //} - - clientCtx, err := cosmosclient.GetClientQueryContext(cmd) - if err != nil { - panic(err) - } - supplierQuerier := suppliertypes.NewQueryClient(clientCtx) - supplierQuery := &suppliertypes.QueryGetSupplierRequest{Address: ""} - - log.Printf("clientCtx: %+v", clientCtx) - _, err = supplierQuerier.Supplier(ctx, supplierQuery) - if err != nil { - panic(err) - } - // Depends on clientCtx, txFactory, EventsQueryClient, & BlockClient. deps, err = supplyTxClient(ctx, deps) if err != nil { @@ -203,12 +181,17 @@ func supplyEventsQueryClient(deps depinject.Config, pocketNodeWebsocketURL strin // TODO_IN_THIS_COMMIT: move func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { - pocketNodeURI, err := cmd.Flags().GetString(cosmosflags.FlagNode) + pocketNodeURLStr, err := cmd.Flags().GetString(cosmosflags.FlagNode) + if err != nil { + return "", err + } + + pocketNodeURL, err := url.Parse(pocketNodeURLStr) if err != nil { return "", err } - return fmt.Sprintf("ws://%s/websocket", pocketNodeURI), nil + return fmt.Sprintf("ws://%s/websocket", pocketNodeURL.Host), nil } func supplyBlockClient( From fa8c2eb77153d4967f557bbe2bd5cb4b73ca2602 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 22:29:01 +0100 Subject: [PATCH 062/127] wip: debugging --- pkg/relayer/cmd/cmd.go | 4 +++- pkg/relayer/proxy/server_builder.go | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 6de57e397..1f31b70ad 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -94,7 +94,9 @@ func runRelayer(cmd *cobra.Command, _ []string) error { // Start the relay miner log.Println("INFO: Starting relay miner...") - relayMiner.Start(ctx) + if err := relayMiner.Start(ctx); err != nil { + return err + } log.Println("INFO: Relay miner stopped; exiting") return nil diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index a5abc47bd..f788045e6 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -13,7 +13,13 @@ import ( // is responsible for listening for incoming relay requests and relaying them to the supported proxied service. func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { // Get the supplier address from the keyring - supplierAddress, err := rp.keyring.Key(rp.signingKeyName) + supplierKey, err := rp.keyring.Key(rp.signingKeyName) + if err != nil { + return err + } + + // TODO_DISCUSS: is there a reason not to assign rp.supplierAddress here? + supplierAddress, err := supplierKey.GetAddress() if err != nil { return err } From 0638e8d1cc1f400a438f171f29f0676b70694940 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 11 Nov 2023 23:07:24 +0100 Subject: [PATCH 063/127] wip: debugging --- Makefile | 15 ++++++++++++--- pkg/relayer/proxy/jsonrpc.go | 14 +++++++++++--- pkg/relayer/proxy/server_builder.go | 8 +++++++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1f06b4f68..ddfaed9fd 100644 --- a/Makefile +++ b/Makefile @@ -362,15 +362,24 @@ supplier_stake: ## Stake tokens for the supplier specified (must specify the APP .PHONY: supplier1_stake supplier1_stake: ## Stake supplier1 - SUPPLIER=supplier1 SERVICES="anvil;http://anvil:8547,svc1;http://localhost:8081" make supplier_stake + # TODO_IMPROVE: consolidate supplier1 staking; why/where is supplier1 + # being staked other than this make target? + # Supplier1 seems to be staked at some point during localnet startup. + # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). + SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8548,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake supplier2_stake: ## Stake supplier2 - SUPPLIER=supplier2 SERVICES="anvil;http://anvil:8547,svc2;http://localhost:8082" make supplier_stake + # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). + SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8548,svc2;http://localhost:8082" make supplier_stake .PHONY: supplier3_stake supplier3_stake: ## Stake supplier3 - SUPPLIER=supplier3 SERVICES="anvil;http://anvil:8547,svc3;http://localhost:8083" make supplier_stake + # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). + SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8548,svc3;http://localhost:8083" make supplier_stake .PHONY: supplier_unstake supplier_unstake: ## Unstake an supplier (must specify the SUPPLIER env var) diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index 8ea953d6f..08a16ae0d 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -48,10 +48,18 @@ func NewJSONRPCServer( servedRelaysProducer chan<- *types.Relay, proxy relayer.RelayerProxy, ) relayer.RelayServer { + // TODO_IN_THIS_COMMIT: refactor / rename / simplify + url, err := url.Parse(supplierEndpoint.Url) + if err != nil { + panic(err) + } + supplierEndpointHost := url.Host + return &jsonRPCServer{ - service: service, - serverEndpoint: supplierEndpoint, - server: &http.Server{Addr: supplierEndpoint.Url}, + service: service, + serverEndpoint: supplierEndpoint, + //server: &http.Server{Addr: supplierEndpoint.Url}, + server: &http.Server{Addr: supplierEndpointHost}, relayerProxy: proxy, proxiedServiceEndpoint: proxiedServiceEndpoint, servedRelaysProducer: servedRelaysProducer, diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index f788045e6..5560a1d4d 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "log" "github.com/pokt-network/poktroll/pkg/relayer" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" @@ -38,11 +39,16 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { for _, serviceConfig := range services { service := serviceConfig.Service proxiedServicesEndpoints := rp.proxiedServicesEndpoints[service.Id] - serviceEndpoints := make([]relayer.RelayServer, len(serviceConfig.Endpoints)) + var serviceEndpoints []relayer.RelayServer for _, endpoint := range serviceConfig.Endpoints { var server relayer.RelayServer + log.Printf( + "INFO: starting relay server for service %s at endpoint %s", + service.Id, endpoint.Url, + ) + // Switch to the RPC type to create the appropriate RelayServer switch endpoint.RpcType { case sharedtypes.RPCType_JSON_RPC: From bb87722d0ba986faa5fe22061796a5b61cde62c2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 14:45:58 +0100 Subject: [PATCH 064/127] revert-or-fixme: add error log lines --- pkg/appgateserver/jsonrpc.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 3be01cca7..b61dc51a1 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -25,6 +25,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Read the request body bytes. payloadBz, err := io.ReadAll(request.Body) if err != nil { + log.Println("ERROR: Failed reading relay request body") return err } @@ -34,6 +35,7 @@ func (app *appGateServer) handleJSONRPCRelay( session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { + log.Println("ERROR: Failed getting current session") return err } log.Printf("DEBUG: Current session ID: %s", session.SessionId) @@ -41,6 +43,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Get a supplier URL and address for the given service and session. supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, sharedtypes.RPCType_JSON_RPC, session) if err != nil { + log.Println("ERROR: Failed getting relayer URL") return err } @@ -56,18 +59,21 @@ func (app *appGateServer) handleJSONRPCRelay( // Get the application's signer. signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { + log.Println("ERROR: Failed getting signer") return err } // Hash and sign the request's signable bytes. signableBz, err := relayRequest.GetSignableBytes() if err != nil { + log.Println("ERROR: Failed getting signable bytes") return err } hash := crypto.Sha256(signableBz) signature, err := signer.Sign(hash) if err != nil { + log.Println("ERROR: Failed signing relay request") return err } relayRequest.Meta.Signature = signature @@ -75,6 +81,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := relayRequest.Marshal() if err != nil { + log.Println("ERROR: Failed marshaling relay request") return err } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) @@ -91,18 +98,22 @@ func (app *appGateServer) handleJSONRPCRelay( log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { + log.Println("ERROR: Failed sending relay request to relayer") return err } // Read the response body bytes. relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { + log.Println("ERROR: Failed reading relay response body") return err } // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} + log.Printf("relayHTTPResponse: %s", string(relayResponseBz)) if err := relayResponse.Unmarshal(relayResponseBz); err != nil { + log.Println("ERROR: Failed unmarshaling relay response") return err } @@ -114,17 +125,20 @@ func (app *appGateServer) handleJSONRPCRelay( // failed responses. log.Println("DEBUG: Verifying signed relay response from...") if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { + log.Println("ERROR: Failed verifying relay response signature") return err } // Marshal the response payload to bytes to be sent back to the application. var responsePayloadBz []byte if _, err = relayResponse.Payload.MarshalTo(responsePayloadBz); err != nil { + log.Println("ERROR: Failed marshaling relay response payload") return err } // Reply with the RelayResponse payload. if _, err := writer.Write(relayRequestBz); err != nil { + log.Println("ERROR: Failed writing relay response payload to writer") return err } From bcfbef1039f2ec8e04530a37d3028f54afc5e2e1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 14:50:03 +0100 Subject: [PATCH 065/127] revert-or-fixme: add debug log lines --- pkg/appgateserver/jsonrpc.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index b61dc51a1..540d16ea1 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -78,6 +78,8 @@ func (app *appGateServer) handleJSONRPCRelay( } relayRequest.Meta.Signature = signature + log.Printf("DEBUG: relayRequest: %+v", relayRequest) + // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := relayRequest.Marshal() if err != nil { @@ -94,6 +96,8 @@ func (app *appGateServer) handleJSONRPCRelay( Body: relayRequestReader, } + log.Printf("DEBUG: relayHTTPRequest: %+v", relayHTTPRequest) + // Perform the HTTP request to the relayer. log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) @@ -111,7 +115,8 @@ func (app *appGateServer) handleJSONRPCRelay( // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} - log.Printf("relayHTTPResponse: %s", string(relayResponseBz)) + log.Printf("DEBUG: relayHTTPResponse body: %s", string(relayResponseBz)) + log.Printf("DEBUG: relayHTTPResponse: %+v", relayHTTPResponse) if err := relayResponse.Unmarshal(relayResponseBz); err != nil { log.Println("ERROR: Failed unmarshaling relay response") return err From 4019a1779b3b559919609ea2dd6f9330b7b5546c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 15:04:42 +0100 Subject: [PATCH 066/127] fix: set relay server handle function --- pkg/relayer/proxy/jsonrpc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index 08a16ae0d..fbbe463e6 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -70,11 +70,15 @@ func NewJSONRPCServer( // It also waits for the passed in context to end before shutting down. // This method is blocking and should be called in a goroutine. func (jsrv *jsonRPCServer) Start(ctx context.Context) error { + go func() { <-ctx.Done() jsrv.server.Shutdown(ctx) }() + // Set the HTTP handler. + jsrv.server.Handler = jsrv + return jsrv.server.ListenAndServe() } From 9c3cfb55d53020d7eb2a8df7bc0e500954ed7aea Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 15:04:55 +0100 Subject: [PATCH 067/127] revert-or-fixme: add debug log lines --- pkg/relayer/proxy/jsonrpc.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index fbbe463e6..5f7228e66 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -98,6 +98,9 @@ func (j *jsonRPCServer) Service() *sharedtypes.Service { // (see https://pkg.go.dev/net/http#Handler) func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { ctx := request.Context() + + log.Printf("DEBUG: ServeHTTP() request: %+v", request) + // Relay the request to the proxied service and build the response that will be sent back to the client. relay, err := jsrv.serveHTTP(ctx, request) if err != nil { @@ -107,6 +110,8 @@ func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.R return } + log.Printf("DEBUG: relay: %+v", relay) + // Send the relay response to the client. if err := jsrv.sendRelayResponse(relay.Res, writer); err != nil { jsrv.replyWithError(writer, err) From 2c130ce72e6d28adeb12a4bcb0d03430f6588e12 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 15:05:15 +0100 Subject: [PATCH 068/127] chore: rename some chan vars --- pkg/relayer/proxy/proxy.go | 6 +++--- pkg/relayer/proxy/server_builder.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index d59ea66a7..c28fd185c 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -72,9 +72,9 @@ type relayerProxy struct { // servedRelays is an observable that notifies the miner about the relays that have been served. servedRelays observable.Observable[*types.Relay] - // servedRelaysProducer is a channel that emits the relays that have been served so that the + // servedRelaysPublishCh is a channel that emits the relays that have been served so that the // servedRelays observable can fan out the notifications to its subscribers. - servedRelaysProducer chan<- *types.Relay + servedRelaysPublishCh chan<- *types.Relay // ringCache is a cache of the public keys used to create the ring for a given application // they are stored in a map of application address to a slice of points on the secp256k1 curve @@ -114,7 +114,7 @@ func NewRelayerProxy( servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() rp.servedRelays = servedRelays - rp.servedRelaysProducer = servedRelaysProducer + rp.servedRelaysPublishCh = servedRelaysProducer rp.accountsQuerier = accounttypes.NewQueryClient(rp.clientCtx) rp.supplierQuerier = suppliertypes.NewQueryClient(rp.clientCtx) rp.sessionQuerier = sessiontypes.NewQueryClient(rp.clientCtx) diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index 5560a1d4d..e28055d69 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -56,7 +56,7 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { service, endpoint, proxiedServicesEndpoints, - rp.servedRelaysProducer, + rp.servedRelaysPublishCh, rp, ) default: From 8493b2c90f40685be7b36d6def370edcf5f2842e Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sun, 12 Nov 2023 16:27:16 +0000 Subject: [PATCH 069/127] feat: fix all bugs, e2e relay works --- pkg/appgateserver/endpoint_selector.go | 2 +- pkg/appgateserver/errors.go | 1 + pkg/appgateserver/jsonrpc.go | 33 +++++++++++++++++--------- pkg/appgateserver/relay_verifier.go | 16 ++++++++++--- pkg/appgateserver/server.go | 2 +- pkg/relayer/protocol/block_heights.go | 4 ++-- pkg/relayer/proxy/error_reply.go | 2 +- pkg/relayer/proxy/jsonrpc.go | 24 ++++++++++++------- pkg/relayer/proxy/proxy.go | 3 +++ pkg/relayer/proxy/relay_builders.go | 18 +++++++++----- pkg/relayer/proxy/relay_verifier.go | 3 +++ pkg/relayer/proxy/rings.go | 17 +++++++++---- pkg/relayer/session/proof.go | 5 ++-- pkg/relayer/session/session.go | 11 +++++---- pkg/relayer/session/sessiontree.go | 1 + proto/pocket/service/relay.proto | 9 +++---- 16 files changed, 102 insertions(+), 49 deletions(-) diff --git a/pkg/appgateserver/endpoint_selector.go b/pkg/appgateserver/endpoint_selector.go index 380c5dad5..c0cb22aee 100644 --- a/pkg/appgateserver/endpoint_selector.go +++ b/pkg/appgateserver/endpoint_selector.go @@ -31,7 +31,7 @@ func (app *appGateServer) getRelayerUrl( if endpoint.RpcType == rpcType { supplierUrl, err := url.Parse(endpoint.Url) if err != nil { - log.Printf("error parsing url: %s", err) + log.Printf("ERROR: error parsing url: %s", err) continue } return supplierUrl, supplier.Address, nil diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index 2c8f281bd..2eab00000 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -10,4 +10,5 @@ var ( ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 4, "missing application address") ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 5, "missing app client signing information") ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 6, "missing app client listening endpoint") + ErrAppGateEmptyRelayResponse = sdkerrors.Register(codespace, 7, "empty relay response") ) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 540d16ea1..6dd6b54ce 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -28,10 +28,16 @@ func (app *appGateServer) handleJSONRPCRelay( log.Println("ERROR: Failed reading relay request body") return err } + log.Printf("DEBUG: relay request body: %s", string(payloadBz)) // Create the relay request payload. relayRequestPayload := &types.RelayRequest_JsonRpcPayload{} - relayRequestPayload.JsonRpcPayload.Unmarshal(payloadBz) + jsonPayload := &types.JSONRPCRequestPayload{} + cdc := types.ModuleCdc + if err := cdc.UnmarshalJSON(payloadBz, jsonPayload); err != nil { + return err + } + relayRequestPayload.JsonRpcPayload = jsonPayload session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { @@ -64,6 +70,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Hash and sign the request's signable bytes. + log.Printf("DEBUG: Signing relay request...") signableBz, err := relayRequest.GetSignableBytes() if err != nil { log.Println("ERROR: Failed getting signable bytes") @@ -78,15 +85,20 @@ func (app *appGateServer) handleJSONRPCRelay( } relayRequest.Meta.Signature = signature - log.Printf("DEBUG: relayRequest: %+v", relayRequest) + // log.Printf("DEBUG: relayRequest: %+v", relayRequest) // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. - relayRequestBz, err := relayRequest.Marshal() + relayRequestBz, err := cdc.Marshal(relayRequest) if err != nil { log.Println("ERROR: Failed marshaling relay request") return err } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) + var relayReq types.RelayRequest + if err := relayReq.Unmarshal(relayRequestBz); err != nil { + return err + } + // log.Printf("DEBUG: Signed relay request: %+v", relayReq) // Create the HTTP request to send the request to the relayer. relayHTTPRequest := &http.Request{ @@ -96,7 +108,7 @@ func (app *appGateServer) handleJSONRPCRelay( Body: relayRequestReader, } - log.Printf("DEBUG: relayHTTPRequest: %+v", relayHTTPRequest) + // log.Printf("DEBUG: relayHTTPRequest: %+v", relayHTTPRequest) // Perform the HTTP request to the relayer. log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) @@ -107,6 +119,7 @@ func (app *appGateServer) handleJSONRPCRelay( } // Read the response body bytes. + log.Printf("DEBUG: Reading relay response body...") relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { log.Println("ERROR: Failed reading relay response body") @@ -115,8 +128,6 @@ func (app *appGateServer) handleJSONRPCRelay( // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} - log.Printf("DEBUG: relayHTTPResponse body: %s", string(relayResponseBz)) - log.Printf("DEBUG: relayHTTPResponse: %+v", relayHTTPResponse) if err := relayResponse.Unmarshal(relayResponseBz); err != nil { log.Println("ERROR: Failed unmarshaling relay response") return err @@ -128,21 +139,21 @@ func (app *appGateServer) handleJSONRPCRelay( // as in some relayer early failures, it may not be signed by the supplier. // TODO_IMPROVE: Add more logging & telemetry so we can get visibility and signal into // failed responses. - log.Println("DEBUG: Verifying signed relay response from...") if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { log.Println("ERROR: Failed verifying relay response signature") return err } // Marshal the response payload to bytes to be sent back to the application. - var responsePayloadBz []byte - if _, err = relayResponse.Payload.MarshalTo(responsePayloadBz); err != nil { - log.Println("ERROR: Failed marshaling relay response payload") + relayResponsePayloadBz, err := cdc.MarshalJSON(relayResponse.GetJsonRpcPayload()) + if err != nil { + log.Println("ERROR: Failed unmarshaling relay response") return err } // Reply with the RelayResponse payload. - if _, err := writer.Write(relayRequestBz); err != nil { + log.Printf("DEBUG: Writing relay response payload: %s", string(relayResponsePayloadBz)) + if _, err := writer.Write(relayResponsePayloadBz); err != nil { log.Println("ERROR: Failed writing relay response payload to writer") return err } diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 712eda7f9..68bf890db 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -2,8 +2,11 @@ package appgateserver import ( "context" + "log" "github.com/cometbft/cometbft/crypto" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" @@ -23,6 +26,10 @@ func (app *appGateServer) verifyResponse( } // Extract the supplier's signature + log.Printf("DEBUG: Verifying relay response signature...") + if relayResponse.Meta == nil { + return ErrAppGateEmptyRelayResponse + } supplierSignature := relayResponse.Meta.SupplierSignature // Get the relay response signable bytes and hash them. @@ -60,12 +67,15 @@ func (app *appGateServer) getSupplierPubKeyFromAddress( } // Unmarshal the query response into a BaseAccount. - account := new(accounttypes.BaseAccount) - if err := account.Unmarshal(accQueryRes.Account.Value); err != nil { + var acc accounttypes.AccountI + reg := codectypes.NewInterfaceRegistry() + accounttypes.RegisterInterfaces(reg) + cdc := codec.NewProtoCodec(reg) + if err := cdc.UnpackAny(accQueryRes.Account, &acc); err != nil { return nil, err } - fetchedPubKey := account.GetPubKey() + fetchedPubKey := acc.GetPubKey() // Cache the retrieved public key. app.supplierAccountCache[supplierAddress] = fetchedPubKey diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index d6410021c..a85d5fae3 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -225,7 +225,7 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ - Id: make([]byte, 0), + Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ // Using conventional error code indicating internal server error. diff --git a/pkg/relayer/protocol/block_heights.go b/pkg/relayer/protocol/block_heights.go index b372376f7..fcf27ec10 100644 --- a/pkg/relayer/protocol/block_heights.go +++ b/pkg/relayer/protocol/block_heights.go @@ -14,7 +14,7 @@ import ( // TODO_TEST(@bryanchriswhite): Add test coverage and more logs func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int64 { createClaimWindowStartBlockHash := createClaimWindowStartBlock.Hash() - log.Printf("using createClaimWindowStartBlock %d's hash %x as randomness", createClaimWindowStartBlock.Height(), createClaimWindowStartBlockHash) + log.Printf("DEBUG: using createClaimWindowStartBlock %d's hash %x as randomness", createClaimWindowStartBlock.Height(), createClaimWindowStartBlockHash) rngSeed, _ := binary.Varint(createClaimWindowStartBlockHash) randomNumber := rand.NewSource(rngSeed).Int63() @@ -32,7 +32,7 @@ func GetEarliestCreateClaimHeight(createClaimWindowStartBlock client.Block) int6 // TODO_TEST(@bryanchriswhite): Add test coverage and more logs func GetEarliestSubmitProofHeight(submitProofWindowStartBlock client.Block) int64 { earliestSubmitProofBlockHash := submitProofWindowStartBlock.Hash() - log.Printf("using submitProofWindowStartBlock %d's hash %x as randomness", submitProofWindowStartBlock.Height(), earliestSubmitProofBlockHash) + log.Printf("DEBUG: using submitProofWindowStartBlock %d's hash %x as randomness", submitProofWindowStartBlock.Height(), earliestSubmitProofBlockHash) rngSeed, _ := binary.Varint(earliestSubmitProofBlockHash) randomNumber := rand.NewSource(rngSeed).Int63() diff --git a/pkg/relayer/proxy/error_reply.go b/pkg/relayer/proxy/error_reply.go index d881057a0..9a268ab95 100644 --- a/pkg/relayer/proxy/error_reply.go +++ b/pkg/relayer/proxy/error_reply.go @@ -16,7 +16,7 @@ func (j *jsonRPCServer) replyWithError(writer http.ResponseWriter, err error) { relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ - Id: make([]byte, 0), + Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ // Using conventional error code indicating internal server error. diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index 5f7228e66..4fa6f5874 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -58,7 +58,7 @@ func NewJSONRPCServer( return &jsonRPCServer{ service: service, serverEndpoint: supplierEndpoint, - //server: &http.Server{Addr: supplierEndpoint.Url}, + // server: &http.Server{Addr: supplierEndpoint.Url}, server: &http.Server{Addr: supplierEndpointHost}, relayerProxy: proxy, proxiedServiceEndpoint: proxiedServiceEndpoint, @@ -70,7 +70,6 @@ func NewJSONRPCServer( // It also waits for the passed in context to end before shutting down. // This method is blocking and should be called in a goroutine. func (jsrv *jsonRPCServer) Start(ctx context.Context) error { - go func() { <-ctx.Done() jsrv.server.Shutdown(ctx) @@ -99,7 +98,7 @@ func (j *jsonRPCServer) Service() *sharedtypes.Service { func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { ctx := request.Context() - log.Printf("DEBUG: ServeHTTP() request: %+v", request) + log.Printf("DEBUG: Serving JSON-RPC relay request...") // Relay the request to the proxied service and build the response that will be sent back to the client. relay, err := jsrv.serveHTTP(ctx, request) @@ -110,8 +109,6 @@ func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.R return } - log.Printf("DEBUG: relay: %+v", relay) - // Send the relay response to the client. if err := jsrv.sendRelayResponse(relay.Res, writer); err != nil { jsrv.replyWithError(writer, err) @@ -134,6 +131,7 @@ func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.R // serveHTTP holds the underlying logic of ServeHTTP. func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) (*types.Relay, error) { // Extract the relay request from the request body. + log.Printf("DEBUG: Extracting relay request from request body...") relayRequest, err := jsrv.newRelayRequest(request) if err != nil { return nil, err @@ -153,19 +151,25 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) // Get the relayRequest payload's `io.ReadCloser` to add it to the http.Request // that will be sent to the proxied (i.e. staked for) service. // (see https://pkg.go.dev/net/http#Request) Body field type. - var payloadBz []byte - if _, err = relayRequest.Payload.MarshalTo(payloadBz); err != nil { + log.Printf("DEBUG: Getting relay request payload...") + cdc := types.ModuleCdc + payloadBz, err := cdc.MarshalJSON(relayRequest.GetJsonRpcPayload()) + if err != nil { return nil, err } requestBodyReader := io.NopCloser(bytes.NewBuffer(payloadBz)) + log.Printf("DEBUG: Relay request payload: %s", string(payloadBz)) // Build the request to be sent to the native service by substituting // the destination URL's host with the native service's listen address. + log.Printf("DEBUG: Building relay request to native service %s...", jsrv.proxiedServiceEndpoint.String()) destinationURL, err := url.Parse(request.URL.String()) if err != nil { return nil, err } - destinationURL.Host = jsrv.proxiedServiceEndpoint.Host + destinationURL.Host = "localhost:8547" + destinationURL.Scheme = "http" + log.Printf("DEBUG: Sending relay request to native service %s...", destinationURL.String()) relayHTTPRequest := &http.Request{ Method: request.Method, @@ -184,6 +188,7 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) // Build the relay response from the native service response // Use relayRequest.Meta.SessionHeader on the relayResponse session header since it was verified to be valid // and has to be the same as the relayResponse session header. + log.Printf("DEBUG: Building relay response from native service response...") relayResponse, err := jsrv.newRelayResponse(httpResponse, relayRequest.Meta.SessionHeader) if err != nil { return nil, err @@ -194,7 +199,8 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) // sendRelayResponse marshals the relay response and sends it to the client. func (j *jsonRPCServer) sendRelayResponse(relayResponse *types.RelayResponse, writer http.ResponseWriter) error { - relayResponseBz, err := relayResponse.Marshal() + cdc := types.ModuleCdc + relayResponseBz, err := cdc.Marshal(relayResponse) if err != nil { return err } diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index c28fd185c..fff7af361 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -118,7 +118,10 @@ func NewRelayerProxy( rp.accountsQuerier = accounttypes.NewQueryClient(rp.clientCtx) rp.supplierQuerier = suppliertypes.NewQueryClient(rp.clientCtx) rp.sessionQuerier = sessiontypes.NewQueryClient(rp.clientCtx) + rp.applicationQuerier = apptypes.NewQueryClient(rp.clientCtx) rp.keyring = rp.clientCtx.Keyring + rp.ringCache = make(map[string][]ringtypes.Point) + rp.ringCacheMutex = &sync.RWMutex{} for _, opt := range opts { opt(rp) diff --git a/pkg/relayer/proxy/relay_builders.go b/pkg/relayer/proxy/relay_builders.go index beb400cad..0628c7c99 100644 --- a/pkg/relayer/proxy/relay_builders.go +++ b/pkg/relayer/proxy/relay_builders.go @@ -2,6 +2,7 @@ package proxy import ( "io" + "log" "net/http" "github.com/pokt-network/poktroll/x/service/types" @@ -15,12 +16,13 @@ func (j *jsonRPCServer) newRelayRequest(request *http.Request) (*types.RelayRequ return nil, err } - var relayRequest types.RelayRequest - if err := relayRequest.Unmarshal(requestBz); err != nil { + log.Printf("DEBUG: Unmarshaling relay request...") + var relayReq types.RelayRequest + if err := relayReq.Unmarshal(requestBz); err != nil { return nil, err } - return &relayRequest, nil + return &relayReq, nil } // newRelayResponse builds a RelayResponse from an http.Response and a SessionHeader. @@ -39,12 +41,16 @@ func (j *jsonRPCServer) newRelayResponse( return nil, err } - jsonRPCResponse := &types.JSONRPCResponsePayload{} - if err := jsonRPCResponse.Unmarshal(responseBz); err != nil { + log.Printf("DEBUG: Unmarshaling relay response...") + relayResponsePayload := &types.RelayResponse_JsonRpcPayload{} + jsonPayload := &types.JSONRPCResponsePayload{} + cdc := types.ModuleCdc + if err := cdc.UnmarshalJSON(responseBz, jsonPayload); err != nil { return nil, err } + relayResponsePayload.JsonRpcPayload = jsonPayload - relayResponse.Payload = &types.RelayResponse_JsonRpcPayload{JsonRpcPayload: jsonRPCResponse} + relayResponse.Payload = &types.RelayResponse_JsonRpcPayload{JsonRpcPayload: jsonPayload} // Sign the relay response and add the signature to the relay response metadata if err = j.relayerProxy.SignRelayResponse(relayResponse); err != nil { diff --git a/pkg/relayer/proxy/relay_verifier.go b/pkg/relayer/proxy/relay_verifier.go index e64955d55..ba8bf7e32 100644 --- a/pkg/relayer/proxy/relay_verifier.go +++ b/pkg/relayer/proxy/relay_verifier.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "log" sdkerrors "cosmossdk.io/errors" ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" @@ -20,6 +21,7 @@ func (rp *relayerProxy) VerifyRelayRequest( service *sharedtypes.Service, ) error { // extract the relay request's ring signature + log.Printf("DEBUG: Verifying relay request signature...") signature := relayRequest.Meta.Signature if signature == nil { return sdkerrors.Wrapf( @@ -73,6 +75,7 @@ func (rp *relayerProxy) VerifyRelayRequest( } // Query for the current session to check if relayRequest sessionId matches the current session. + log.Printf("DEBUG: Verifying relay request session...") currentBlock := rp.blockClient.LatestBlock(ctx) sessionQuery := &sessiontypes.QueryGetSessionRequest{ ApplicationAddress: appAddress, diff --git a/pkg/relayer/proxy/rings.go b/pkg/relayer/proxy/rings.go index 59a19ae70..86651247d 100644 --- a/pkg/relayer/proxy/rings.go +++ b/pkg/relayer/proxy/rings.go @@ -6,9 +6,12 @@ package proxy import ( "context" "fmt" + "log" ring_secp256k1 "github.com/athanorlabs/go-dleq/secp256k1" ringtypes "github.com/athanorlabs/go-dleq/types" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" ring "github.com/noot/ring-go" @@ -28,7 +31,10 @@ func (rp *relayerProxy) getRingForAppAddress(ctx context.Context, appAddress str var err error if !ok { // if the ring is not in the cache, get it from the application module + log.Printf("DEBUG: Ring not found in cache for %s, fetching from application module...", appAddress) points, err = rp.getDelegatedPubKeysForAddress(ctx, appAddress) + } else { + log.Printf("DEBUG: Ring found in cache for %s", appAddress) } if err != nil { return nil, err @@ -50,8 +56,8 @@ func (rp *relayerProxy) getDelegatedPubKeysForAddress( ctx context.Context, appAddress string, ) ([]ringtypes.Point, error) { - rp.ringCacheMutex.RLock() - defer rp.ringCacheMutex.RUnlock() + rp.ringCacheMutex.Lock() + defer rp.ringCacheMutex.Unlock() // get the application's on chain state req := &apptypes.QueryGetApplicationRequest{Address: appAddress} @@ -101,8 +107,11 @@ func (rp *relayerProxy) addressesToPoints(ctx context.Context, addresses []strin if err != nil { return nil, fmt.Errorf("unable to get account for address: %s [%w]", addr, err) } - acc := new(accounttypes.BaseAccount) - if err := acc.Unmarshal(pubKeyRes.Account.Value); err != nil { + var acc accounttypes.AccountI + reg := codectypes.NewInterfaceRegistry() + accounttypes.RegisterInterfaces(reg) + cdc := codec.NewProtoCodec(reg) + if err := cdc.UnpackAny(pubKeyRes.Account, &acc); err != nil { return nil, fmt.Errorf("unable to deserialise account for address: %s [%w]", addr, err) } key := acc.GetPubKey() diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 4a7c415aa..5d54a4338 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -30,8 +30,7 @@ func (rs *relayerSessionsManager) submitProofs( rs.mapWaitForEarliestSubmitProofHeight, ) - failedSubmitProofSessionsObs, failedSubmitProofSessionsPublishCh := - channel.NewObservable[relayer.SessionTree]() + failedSubmitProofSessionsObs, failedSubmitProofSessionsPublishCh := channel.NewObservable[relayer.SessionTree]() // Map sessionsWithOpenProofWindow to a new observable of an either type, // populated with the session or an error, which is notified after the session @@ -74,7 +73,7 @@ func (rs *relayerSessionsManager) waitForEarliestSubmitProofHeight( // + claimproofparams.GovSubmitProofWindowStartHeightOffset // we wait for submitProofWindowStartHeight to be received before proceeding since we need its hash - log.Printf("waiting and blocking for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) + log.Printf("INFO: waiting and blocking for global earliest proof submission submitProofWindowStartBlock height: %d", submitProofWindowStartHeight) submitProofWindowStartBlock := rs.waitForBlock(ctx, submitProofWindowStartHeight) earliestSubmitProofHeight := protocol.GetEarliestSubmitProofHeight(submitProofWindowStartBlock) diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index dc2544f43..7f3f7e188 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -51,7 +51,8 @@ func NewRelayerSessions( opts ...relayer.RelayerSessionsManagerOption, ) (relayer.RelayerSessionsManager, error) { rs := &relayerSessionsManager{ - sessionsTrees: make(sessionsTreesMap), + sessionsTrees: make(sessionsTreesMap), + sessionsTreesMu: &sync.Mutex{}, } if err := depinject.Inject( @@ -170,7 +171,7 @@ func (rs *relayerSessionsManager) removeFromRelayerSessions(sessionHeader *sessi sessionsTreesEndingAtBlockHeight, ok := rs.sessionsTrees[sessionHeader.SessionEndBlockHeight] if !ok { - log.Printf("no session tree found for sessions ending at height %d", sessionHeader.SessionEndBlockHeight) + log.Printf("DEBUG: no session tree found for sessions ending at height %d", sessionHeader.SessionEndBlockHeight) return } @@ -215,16 +216,18 @@ func (rs *relayerSessionsManager) mapAddRelayToSessionTree( _ context.Context, relay *relayer.MinedRelay, ) (_ error, skip bool) { + rs.sessionsTreesMu.Lock() + defer rs.sessionsTreesMu.Unlock() // ensure the session tree exists for this relay sessionHeader := relay.GetReq().GetMeta().GetSessionHeader() smst, err := rs.ensureSessionTree(sessionHeader) if err != nil { - log.Printf("failed to ensure session tree: %s\n", err) + log.Printf("ERROR: failed to ensure session tree: %s\n", err) return err, false } if err := smst.Update(relay.Hash, relay.Bytes, 1); err != nil { - log.Printf("failed to update smt: %s\n", err) + log.Printf("ERROR: failed to update smt: %s\n", err) return err, false } diff --git a/pkg/relayer/session/sessiontree.go b/pkg/relayer/session/sessiontree.go index fb27ff307..2ae0cfb52 100644 --- a/pkg/relayer/session/sessiontree.go +++ b/pkg/relayer/session/sessiontree.go @@ -86,6 +86,7 @@ func NewSessionTree( storePath: storePath, treeStore: treeStore, tree: tree, + sessionMu: &sync.Mutex{}, removeFromRelayerSessions: removeFromRelayerSessions, } diff --git a/proto/pocket/service/relay.proto b/proto/pocket/service/relay.proto index 7c9e888e4..7715a41c1 100644 --- a/proto/pocket/service/relay.proto +++ b/proto/pocket/service/relay.proto @@ -37,10 +37,11 @@ message RelayRequest { // JSONRPCRequestPayload contains the payload for a JSON-RPC request. // See https://www.jsonrpc.org/specification#request_object for more details. message JSONRPCRequestPayload { - bytes id = 1; // Identifier established by the Client to create context for the request. + uint32 id = 1; // Identifier established by the Client to create context for the request. string jsonrpc = 2; // Version of JSON-RPC. Must be exactly "2.0". string method = 3; // Method being invoked on the server. - map parameters = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures + //map params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures + repeated string params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures } // RESTRequestType represents the type of REST request. @@ -82,9 +83,9 @@ message RelayResponseMetadata { // JSONRPCResponsePayload contains the response details for a JSON-RPC relay. // See www.jsonrpc.org/specification for more details. message JSONRPCResponsePayload { - bytes id = 1; // Identifier established by the Client to link the response back to the request. + uint32 id = 1; // Identifier established by the Client to link the response back to the request. string jsonrpc = 2; // Version of JSON-RPC. Must be exactly "2.0". - bytes result = 3; // Response result payload. + string result = 3; // Response result payload. JSONRPCResponseError error = 4; // Error message, if any. Can be nil. } From dfd28f8f3257f85ac97acc6c44277ad3d99f8a7b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sun, 12 Nov 2023 17:34:07 +0000 Subject: [PATCH 070/127] chore: add some todo comments --- pkg/relayer/proxy/jsonrpc.go | 5 ++++- proto/pocket/service/relay.proto | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index 4fa6f5874..bf7736f33 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -162,13 +162,16 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) // Build the request to be sent to the native service by substituting // the destination URL's host with the native service's listen address. + // TODO_HACK: Currently, the native service's listen address is hardcoded to localhost:8547. + // This should be changed to the actual listen address of the native service. + // However, the native service 'ws://anvil:8547' causes errors log.Printf("DEBUG: Building relay request to native service %s...", jsrv.proxiedServiceEndpoint.String()) destinationURL, err := url.Parse(request.URL.String()) if err != nil { return nil, err } destinationURL.Host = "localhost:8547" - destinationURL.Scheme = "http" + destinationURL.Scheme = "http" // this is not captured and needs to be set log.Printf("DEBUG: Sending relay request to native service %s...", destinationURL.String()) relayHTTPRequest := &http.Request{ diff --git a/proto/pocket/service/relay.proto b/proto/pocket/service/relay.proto index 7715a41c1..968fd8dfe 100644 --- a/proto/pocket/service/relay.proto +++ b/proto/pocket/service/relay.proto @@ -40,6 +40,8 @@ message JSONRPCRequestPayload { uint32 id = 1; // Identifier established by the Client to create context for the request. string jsonrpc = 2; // Version of JSON-RPC. Must be exactly "2.0". string method = 3; // Method being invoked on the server. + // TODO_RESEARCH: Find out why the params being a map causes errors + // should they be a list of maps? //map params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures repeated string params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures } From 5ab0e4c93e9c27644f30cf814dbfbea325572b30 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sun, 12 Nov 2023 22:14:15 +0100 Subject: [PATCH 071/127] wip: debugging --- pkg/relayer/session/session.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 7f3f7e188..c8f2ad30b 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -115,9 +115,6 @@ func (rs *relayerSessionsManager) InsertRelays(relays observable.Observable[*rel // ensureSessionTree returns the SessionTree for a given session. // If no tree for the session exists, a new SessionTree is created before returning. func (rs *relayerSessionsManager) ensureSessionTree(sessionHeader *sessiontypes.SessionHeader) (relayer.SessionTree, error) { - rs.sessionsTreesMu.Lock() - defer rs.sessionsTreesMu.Unlock() - sessionsTrees, ok := rs.sessionsTrees[sessionHeader.SessionEndBlockHeight] // If there is no map for sessions at the sessionEndHeight, create one. @@ -130,8 +127,9 @@ func (rs *relayerSessionsManager) ensureSessionTree(sessionHeader *sessiontypes. sessionTree, ok := sessionsTrees[sessionHeader.SessionId] // If the sessionTree does not exist, create it. + var err error if !ok { - sessionTree, err := NewSessionTree(sessionHeader, rs.storesDirectory, rs.removeFromRelayerSessions) + sessionTree, err = NewSessionTree(sessionHeader, rs.storesDirectory, rs.removeFromRelayerSessions) if err != nil { return nil, err } @@ -148,6 +146,9 @@ func (rs *relayerSessionsManager) mapBlockToSessionsToClaim( _ context.Context, block client.Block, ) (sessionTrees []relayer.SessionTree, skip bool) { + rs.sessionsTreesMu.Lock() + defer rs.sessionsTreesMu.Unlock() + // Check if there are sessions that need to enter the claim/proof phase // as their end block height was the one before the last committed block. // Iterate over the sessionsTrees map to get the ones that end at a block height From ff4997a56f47575e03593eb7eb408ddc6cbb85db Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 13 Nov 2023 10:00:08 +0100 Subject: [PATCH 072/127] fix: use remote helm charts again --- Tiltfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tiltfile b/Tiltfile index bc470c037..b8c049749 100644 --- a/Tiltfile +++ b/Tiltfile @@ -11,7 +11,7 @@ localnet_config_defaults = { "gateways": {"count": 1}, # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development - "helm_chart_local_repo": {"enabled": True, "path": "../helm-charts"}, + "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, } localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) localnet_config = {} From ac54c247b85905d8220a7577c4b4d69dbbb835a0 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Mon, 13 Nov 2023 11:42:03 +0100 Subject: [PATCH 073/127] fix: put adequate proxied services endpoitns, prevent session republishing --- .gitignore | 3 +++ config.yml | 2 +- pkg/relayer/cmd/cmd.go | 2 +- pkg/relayer/proxy/jsonrpc.go | 11 ++--------- pkg/relayer/session/session.go | 6 +++++- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 6dbdd3c40..5f6cea3d2 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,7 @@ localnet_config.yaml # Relase artifacts produced by `ignite chain build --release` release +# SMT KVStore files +smt + go.work.sum diff --git a/config.yml b/config.yml index f21fe2be6..27af27ff8 100644 --- a/config.yml +++ b/config.yml @@ -100,7 +100,7 @@ genesis: - endpoints: - configs: [] rpc_type: JSON_RPC - url: http://anvil:8547 + url: http://localhost:8548 service: id: anvil name: "" diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 1f31b70ad..225d52789 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -267,7 +267,7 @@ func supplyRelayerProxy( cmd *cobra.Command, ) (depinject.Config, error) { // TODO_TECHDEBT: this should be populated from some relayerProxy config. - anvilURL, err := url.Parse("ws://anvil:8547/") + anvilURL, err := url.Parse("http://localhost:8547/") if err != nil { return nil, err } diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index bf7736f33..a037bd567 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -162,23 +162,16 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) // Build the request to be sent to the native service by substituting // the destination URL's host with the native service's listen address. - // TODO_HACK: Currently, the native service's listen address is hardcoded to localhost:8547. - // This should be changed to the actual listen address of the native service. - // However, the native service 'ws://anvil:8547' causes errors log.Printf("DEBUG: Building relay request to native service %s...", jsrv.proxiedServiceEndpoint.String()) - destinationURL, err := url.Parse(request.URL.String()) if err != nil { return nil, err } - destinationURL.Host = "localhost:8547" - destinationURL.Scheme = "http" // this is not captured and needs to be set - log.Printf("DEBUG: Sending relay request to native service %s...", destinationURL.String()) relayHTTPRequest := &http.Request{ Method: request.Method, Header: request.Header, - URL: destinationURL, - Host: destinationURL.Host, + URL: &jsrv.proxiedServiceEndpoint, + Host: jsrv.proxiedServiceEndpoint.Host, Body: requestBodyReader, } diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index c8f2ad30b..0a0f66aff 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -154,7 +154,11 @@ func (rs *relayerSessionsManager) mapBlockToSessionsToClaim( // Iterate over the sessionsTrees map to get the ones that end at a block height // lower than the current block height. for endBlockHeight, sessionsTreesEndingAtBlockHeight := range rs.sessionsTrees { - if endBlockHeight < block.Height() { + // TODO: We need this to be == instead of <= because we don't want to keep sending + // the same session while waiting the next step. This does not address the case + // where the block client misses the target block which should be handled by the + // retry mechanism. + if endBlockHeight == block.Height() { // Iterate over the sessionsTrees that end at this block height (or // less) and add them to the list of sessionTrees to be published. for _, sessionTree := range sessionsTreesEndingAtBlockHeight { From efc2ab5a2a724b5e4385114ca2e4f1fe6950086e Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Mon, 13 Nov 2023 21:39:32 +0100 Subject: [PATCH 074/127] chore: Refactor JSONRPCServer and server builder --- pkg/relayer/proxy/jsonrpc.go | 19 +++---------------- pkg/relayer/proxy/server_builder.go | 9 ++++++++- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index a037bd567..a07cb118a 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -19,10 +19,6 @@ type jsonRPCServer struct { // service is the service that the server is responsible for. service *sharedtypes.Service - // serverEndpoint is the advertised endpoint configuration that the server uses to - // listen for incoming relay requests. - serverEndpoint *sharedtypes.SupplierEndpoint - // proxiedServiceEndpoint is the address of the proxied service that the server relays requests to. proxiedServiceEndpoint url.URL @@ -43,22 +39,13 @@ type jsonRPCServer struct { // a RelayServer that listens to incoming RelayRequests. func NewJSONRPCServer( service *sharedtypes.Service, - supplierEndpoint *sharedtypes.SupplierEndpoint, + supplierEndpointHost string, proxiedServiceEndpoint url.URL, servedRelaysProducer chan<- *types.Relay, proxy relayer.RelayerProxy, ) relayer.RelayServer { - // TODO_IN_THIS_COMMIT: refactor / rename / simplify - url, err := url.Parse(supplierEndpoint.Url) - if err != nil { - panic(err) - } - supplierEndpointHost := url.Host - return &jsonRPCServer{ - service: service, - serverEndpoint: supplierEndpoint, - // server: &http.Server{Addr: supplierEndpoint.Url}, + service: service, server: &http.Server{Addr: supplierEndpointHost}, relayerProxy: proxy, proxiedServiceEndpoint: proxiedServiceEndpoint, @@ -121,7 +108,7 @@ func (jsrv *jsonRPCServer) ServeHTTP(writer http.ResponseWriter, request *http.R relay.Res.Meta.SessionHeader.ApplicationAddress, relay.Res.Meta.SessionHeader.Service.Id, relay.Res.Meta.SessionHeader.SessionStartBlockHeight, - jsrv.serverEndpoint.Url, + jsrv.server.Addr, ) // Emit the relay to the servedRelays observable. diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index e28055d69..4238c1ce3 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -3,6 +3,7 @@ package proxy import ( "context" "log" + "net/url" "github.com/pokt-network/poktroll/pkg/relayer" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" @@ -42,6 +43,12 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { var serviceEndpoints []relayer.RelayServer for _, endpoint := range serviceConfig.Endpoints { + url, err := url.Parse(endpoint.Url) + if err != nil { + return err + } + supplierEndpointHost := url.Host + var server relayer.RelayServer log.Printf( @@ -54,7 +61,7 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { case sharedtypes.RPCType_JSON_RPC: server = NewJSONRPCServer( service, - endpoint, + supplierEndpointHost, proxiedServicesEndpoints, rp.servedRelaysPublishCh, rp, From 9c326751ee82047c73741ba91160cd1317c96745 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 13 Nov 2023 20:03:42 -0800 Subject: [PATCH 075/127] Upate a couple small comments in the maketfile --- .gitignore | 6 ++++++ Makefile | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5f6cea3d2..b6ff1eab7 100644 --- a/.gitignore +++ b/.gitignore @@ -65,4 +65,10 @@ release # SMT KVStore files smt +# Do not allow a multi-moduled projected go.work.sum + +# TODO_IN_THIS_COMMIT: Why did we start generating .dot files? +# **/*.dot + + diff --git a/Makefile b/Makefile index ddfaed9fd..395ebc5e9 100644 --- a/Makefile +++ b/Makefile @@ -281,7 +281,7 @@ app_stake: ## Stake tokens for the application specified (must specify the APP a poktrolld --home=$(POKTROLLD_HOME) tx application stake-application 1000upokt $(SERVICES) --keyring-backend test --from $(APP) --node $(POCKET_NODE) .PHONY: app1_stake -app1_stake: ## Stake app1 +app1_stake: ## Stake app1 (also staked in genesis) APP=app1 SERVICES=anvil,svc1,svc2 make app_stake .PHONY: app2_stake @@ -361,7 +361,7 @@ supplier_stake: ## Stake tokens for the supplier specified (must specify the APP poktrolld --home=$(POKTROLLD_HOME) tx supplier stake-supplier 1000upokt "$(SERVICES)" --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) .PHONY: supplier1_stake -supplier1_stake: ## Stake supplier1 +supplier1_stake: ## Stake supplier1 (also staked in genesis) # TODO_IMPROVE: consolidate supplier1 staking; why/where is supplier1 # being staked other than this make target? # Supplier1 seems to be staked at some point during localnet startup. From 17597c48fed20916add58dc5a25664d1340be40d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:04:26 +0100 Subject: [PATCH 076/127] revert: comment relayers out of tiltfile --- Tiltfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Tiltfile b/Tiltfile index b8c049749..8e9ab4e82 100644 --- a/Tiltfile +++ b/Tiltfile @@ -122,16 +122,16 @@ helm_resource( image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], ) -# helm_resource( -# "relayers", -# poktroll_chart, -# flags=[ -# "--values=./localnet/kubernetes/values-common.yaml", -# "--set=replicaCount=" + str(localnet_config["relayers"]["count"]), -# ], -# image_deps=["poktrolld"], -# image_keys=[("image.repository", "image.tag")], -# ) +helm_resource( + "relayers", + poktroll_chart, + flags=[ + "--values=./localnet/kubernetes/values-common.yaml", + "--set=replicaCount=" + str(localnet_config["relayers"]["count"]), + ], + image_deps=["poktrolld"], + image_keys=[("image.repository", "image.tag")], +) # Configure tilt resources (tilt labels and port forwards) for all of the nodes above k8s_resource( @@ -145,10 +145,10 @@ k8s_resource( resource_deps=["celestia-rollkit"], port_forwards=["36657", "40004"], ) -# k8s_resource( -# "relayers", -# labels=["blockchains"], -# resource_deps=["sequencer"], -# port_forwards=["8545", "8546", "40005"], -# ) +k8s_resource( + "relayers", + labels=["blockchains"], + resource_deps=["sequencer"], + port_forwards=["8545", "8546", "40005"], +) k8s_resource("anvil", labels=["blockchains"], port_forwards=["8547"]) From 23ac5512980810c3d5f58363a9c107c6c630ad5e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:34:22 +0100 Subject: [PATCH 077/127] chore: fix subcmd name `relayerminer` -> `relayminer` --- pkg/relayer/cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 225d52789..150877c5c 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -33,7 +33,7 @@ var ( func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "relayerminer", + Use: "relayminer", Short: "Run a relay miner", Long: `Run a relay miner`, RunE: runRelayer, From e13211e36d9dca16c5cf649623adfdb3acb70bc0 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:34:35 +0100 Subject: [PATCH 078/127] chore: improve logging --- pkg/relayer/session/claim.go | 3 +++ pkg/relayer/session/proof.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/relayer/session/claim.go b/pkg/relayer/session/claim.go index 712e7a9e5..9401b4af7 100644 --- a/pkg/relayer/session/claim.go +++ b/pkg/relayer/session/claim.go @@ -106,6 +106,9 @@ func (rs *relayerSessionsManager) newMapClaimSessionFn( return either.Error[relayer.SessionTree](err), false } + latestBlock := rs.blockClient.LatestBlock(ctx) + log.Printf("INFO: currentBlock: %d, submitting claim", latestBlock.Height()+1) + sessionHeader := session.GetSessionHeader() if err := rs.supplierClient.CreateClaim(ctx, *sessionHeader, claimRoot); err != nil { failedCreateClaimSessionsPublishCh <- session diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 5d54a4338..00f9b3df5 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -99,7 +99,7 @@ func (rs *relayerSessionsManager) newMapProveSessionFn( return either.Error[relayer.SessionTree](err), false } - log.Printf("currentBlock: %d, submitting proof", latestBlock.Height()+1) + log.Printf("INFO: currentBlock: %d, submitting proof", latestBlock.Height()+1) // SubmitProof ensures on-chain proof inclusion so we can safely prune the tree. if err := rs.supplierClient.SubmitProof( ctx, From cf5910fe1cdda26ad1fedd8215b798947913681e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:50:52 +0100 Subject: [PATCH 079/127] chore: cleanup error messaging & logging in appgate server --- pkg/appgateserver/errors.go | 1 + pkg/appgateserver/jsonrpc.go | 49 +++++++++++------------------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index 2eab00000..eecf99f44 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -11,4 +11,5 @@ var ( ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 5, "missing app client signing information") ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 6, "missing app client listening endpoint") ErrAppGateEmptyRelayResponse = sdkerrors.Register(codespace, 7, "empty relay response") + ErrAppGateHandleRelay = sdkerrors.Register(codespace, 8, "internal error handling relay request") ) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 6dd6b54ce..5c778cde8 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -25,8 +25,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Read the request body bytes. payloadBz, err := io.ReadAll(request.Body) if err != nil { - log.Println("ERROR: Failed reading relay request body") - return err + return ErrAppGateHandleRelay.Wrapf("reading relay request body", err) } log.Printf("DEBUG: relay request body: %s", string(payloadBz)) @@ -41,16 +40,14 @@ func (app *appGateServer) handleJSONRPCRelay( session, err := app.getCurrentSession(ctx, appAddress, serviceId) if err != nil { - log.Println("ERROR: Failed getting current session") - return err + return ErrAppGateHandleRelay.Wrapf("getting current session: %s", err) } log.Printf("DEBUG: Current session ID: %s", session.SessionId) // Get a supplier URL and address for the given service and session. supplierUrl, supplierAddress, err := app.getRelayerUrl(ctx, serviceId, sharedtypes.RPCType_JSON_RPC, session) if err != nil { - log.Println("ERROR: Failed getting relayer URL") - return err + return ErrAppGateHandleRelay.Wrapf("getting supplier URL: %s", err) } // Create the relay request. @@ -65,40 +62,32 @@ func (app *appGateServer) handleJSONRPCRelay( // Get the application's signer. signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { - log.Println("ERROR: Failed getting signer") - return err + return ErrAppGateHandleRelay.Wrapf("getting signer", err) } // Hash and sign the request's signable bytes. - log.Printf("DEBUG: Signing relay request...") signableBz, err := relayRequest.GetSignableBytes() if err != nil { - log.Println("ERROR: Failed getting signable bytes") - return err + return ErrAppGateHandleRelay.Wrapf("getting signable bytes", err) } hash := crypto.Sha256(signableBz) signature, err := signer.Sign(hash) if err != nil { - log.Println("ERROR: Failed signing relay request") - return err + return ErrAppGateHandleRelay.Wrapf("signing relay", err) } relayRequest.Meta.Signature = signature - // log.Printf("DEBUG: relayRequest: %+v", relayRequest) - // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := cdc.Marshal(relayRequest) if err != nil { - log.Println("ERROR: Failed marshaling relay request") - return err + return ErrAppGateHandleRelay.Wrapf("marshalling relay request", err) } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) var relayReq types.RelayRequest if err := relayReq.Unmarshal(relayRequestBz); err != nil { - return err + return ErrAppGateHandleRelay.Wrapf("unmarshalling relay response", err) } - // log.Printf("DEBUG: Signed relay request: %+v", relayReq) // Create the HTTP request to send the request to the relayer. relayHTTPRequest := &http.Request{ @@ -108,29 +97,23 @@ func (app *appGateServer) handleJSONRPCRelay( Body: relayRequestReader, } - // log.Printf("DEBUG: relayHTTPRequest: %+v", relayHTTPRequest) - // Perform the HTTP request to the relayer. log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { - log.Println("ERROR: Failed sending relay request to relayer") - return err + return ErrAppGateHandleRelay.Wrapf("sending relay request", err) } // Read the response body bytes. - log.Printf("DEBUG: Reading relay response body...") relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { - log.Println("ERROR: Failed reading relay response body") - return err + return ErrAppGateHandleRelay.Wrapf("reading relay response body", err) } // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} if err := relayResponse.Unmarshal(relayResponseBz); err != nil { - log.Println("ERROR: Failed unmarshaling relay response") - return err + return ErrAppGateHandleRelay.Wrapf("unmarshalling relay response", err) } // Verify the response signature. We use the supplier address that we got from @@ -140,22 +123,20 @@ func (app *appGateServer) handleJSONRPCRelay( // TODO_IMPROVE: Add more logging & telemetry so we can get visibility and signal into // failed responses. if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { - log.Println("ERROR: Failed verifying relay response signature") - return err + // TODO_DISCUSS: should this be its own error type and asserted against in tests? + return ErrAppGateHandleRelay.Wrapf("verifying relay response signature", err) } // Marshal the response payload to bytes to be sent back to the application. relayResponsePayloadBz, err := cdc.MarshalJSON(relayResponse.GetJsonRpcPayload()) if err != nil { - log.Println("ERROR: Failed unmarshaling relay response") - return err + return ErrAppGateHandleRelay.Wrapf("unmarshallig relay response", err) } // Reply with the RelayResponse payload. log.Printf("DEBUG: Writing relay response payload: %s", string(relayResponsePayloadBz)) if _, err := writer.Write(relayResponsePayloadBz); err != nil { - log.Println("ERROR: Failed writing relay response payload to writer") - return err + return ErrAppGateHandleRelay.Wrapf("writing relay response payload", err) } return nil From eb56997ca6b1a7fe951c70f61f50ebc3dc1564cc Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:52:46 +0100 Subject: [PATCH 080/127] refactor: rename misnamed `jsonRPCServer` receiver var --- pkg/relayer/proxy/error_reply.go | 2 +- pkg/relayer/proxy/jsonrpc.go | 6 +++--- pkg/relayer/proxy/relay_builders.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/relayer/proxy/error_reply.go b/pkg/relayer/proxy/error_reply.go index 9a268ab95..c6c5606e6 100644 --- a/pkg/relayer/proxy/error_reply.go +++ b/pkg/relayer/proxy/error_reply.go @@ -12,7 +12,7 @@ import ( // the caller pass it along with the error if available. // TODO_TECHDEBT: This method should be aware of the nature of the error to use the appropriate JSONRPC // Code, Message and Data. Possibly by augmenting the passed in error with the adequate information. -func (j *jsonRPCServer) replyWithError(writer http.ResponseWriter, err error) { +func (jsrv *jsonRPCServer) replyWithError(writer http.ResponseWriter, err error) { relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ diff --git a/pkg/relayer/proxy/jsonrpc.go b/pkg/relayer/proxy/jsonrpc.go index a07cb118a..113d6285e 100644 --- a/pkg/relayer/proxy/jsonrpc.go +++ b/pkg/relayer/proxy/jsonrpc.go @@ -74,8 +74,8 @@ func (jsrv *jsonRPCServer) Stop(ctx context.Context) error { } // Service returns the JSON-RPC service. -func (j *jsonRPCServer) Service() *sharedtypes.Service { - return j.service +func (jsrv *jsonRPCServer) Service() *sharedtypes.Service { + return jsrv.service } // ServeHTTP listens for incoming relay requests. It implements the respective @@ -181,7 +181,7 @@ func (jsrv *jsonRPCServer) serveHTTP(ctx context.Context, request *http.Request) } // sendRelayResponse marshals the relay response and sends it to the client. -func (j *jsonRPCServer) sendRelayResponse(relayResponse *types.RelayResponse, writer http.ResponseWriter) error { +func (jsrv *jsonRPCServer) sendRelayResponse(relayResponse *types.RelayResponse, writer http.ResponseWriter) error { cdc := types.ModuleCdc relayResponseBz, err := cdc.Marshal(relayResponse) if err != nil { diff --git a/pkg/relayer/proxy/relay_builders.go b/pkg/relayer/proxy/relay_builders.go index 0628c7c99..c2e9775c4 100644 --- a/pkg/relayer/proxy/relay_builders.go +++ b/pkg/relayer/proxy/relay_builders.go @@ -10,7 +10,7 @@ import ( ) // newRelayRequest builds a RelayRequest from an http.Request. -func (j *jsonRPCServer) newRelayRequest(request *http.Request) (*types.RelayRequest, error) { +func (jsrv *jsonRPCServer) newRelayRequest(request *http.Request) (*types.RelayRequest, error) { requestBz, err := io.ReadAll(request.Body) if err != nil { return nil, err @@ -28,7 +28,7 @@ func (j *jsonRPCServer) newRelayRequest(request *http.Request) (*types.RelayRequ // newRelayResponse builds a RelayResponse from an http.Response and a SessionHeader. // It also signs the RelayResponse and assigns it to RelayResponse.Meta.SupplierSignature. // If the response has a non-nil body, it will be parsed as a JSONRPCResponsePayload. -func (j *jsonRPCServer) newRelayResponse( +func (jsrv *jsonRPCServer) newRelayResponse( response *http.Response, sessionHeader *sessiontypes.SessionHeader, ) (*types.RelayResponse, error) { @@ -53,7 +53,7 @@ func (j *jsonRPCServer) newRelayResponse( relayResponse.Payload = &types.RelayResponse_JsonRpcPayload{JsonRpcPayload: jsonPayload} // Sign the relay response and add the signature to the relay response metadata - if err = j.relayerProxy.SignRelayResponse(relayResponse); err != nil { + if err = jsrv.relayerProxy.SignRelayResponse(relayResponse); err != nil { return nil, err } From 5e6038126623e5aa42e66b2930e047d03d39f2a7 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:57:34 +0100 Subject: [PATCH 081/127] chore: remove appgate server debug log --- pkg/appgateserver/relay_verifier.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 68bf890db..4f86e1cdd 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -2,7 +2,6 @@ package appgateserver import ( "context" - "log" "github.com/cometbft/cometbft/crypto" "github.com/cosmos/cosmos-sdk/codec" @@ -26,7 +25,6 @@ func (app *appGateServer) verifyResponse( } // Extract the supplier's signature - log.Printf("DEBUG: Verifying relay response signature...") if relayResponse.Meta == nil { return ErrAppGateEmptyRelayResponse } From 0bd05ae7f250b80c5dd05b679abe7fbb3c5a7daf Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 11:58:54 +0100 Subject: [PATCH 082/127] chore: unexport `relayMiner` struct --- pkg/relayer/cmd/cmd.go | 4 ++-- pkg/relayer/relayminer.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 150877c5c..912494c83 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -69,7 +69,7 @@ func runRelayer(cmd *cobra.Command, _ []string) error { defer cancelCtx() // Sets up the following dependencies: - // Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, RelayMiner dependencies. + // Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, relayMiner dependencies. deps, err := setupRelayerDependencies(ctx, cmd) if err != nil { return err @@ -103,7 +103,7 @@ func runRelayer(cmd *cobra.Command, _ []string) error { } // setupRelayerDependencies sets up all the dependencies the relay miner needs to run: -// Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, RelayMiner +// Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, relayMiner func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index 4832ef56c..81ff82256 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -7,9 +7,9 @@ import ( "cosmossdk.io/depinject" ) -// RelayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). +// relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). // It starts and stops the RelayerProxy and provide the served relays observable to them miner. -type RelayMiner struct { +type relayMiner struct { relayerProxy RelayerProxy miner Miner relayerSessionsManager RelayerSessionsManager @@ -17,8 +17,8 @@ type RelayMiner struct { // NewRelayMiner creates a new Relayer instance with the given dependencies. // It injects the dependencies into the Relayer instance and returns it. -func NewRelayMiner(ctx context.Context, deps depinject.Config) (*RelayMiner, error) { - rel := &RelayMiner{} +func NewRelayMiner(ctx context.Context, deps depinject.Config) (*relayMiner, error) { + rel := &relayMiner{} if err := depinject.Inject( deps, @@ -40,7 +40,7 @@ func NewRelayMiner(ctx context.Context, deps depinject.Config) (*RelayMiner, err // Start provides the miner with the served relays observable and starts the relayer proxy. // This method is blocking while the relayer proxy is running and returns when Stop is called // or when the relayer proxy fails to start. -func (rel *RelayMiner) Start(ctx context.Context) error { +func (rel *relayMiner) Start(ctx context.Context) error { // relayerSessionsManager.Start does not block. // Set up the session (proof/claim) lifecycle pipeline. log.Println("INFO: Starting relayer sessions manager...") @@ -60,6 +60,6 @@ func (rel *RelayMiner) Start(ctx context.Context) error { // Stop stops the relayer proxy which in turn stops all advertised relay servers // and unsubscribes the miner from the served relays observable. -func (rel *RelayMiner) Stop(ctx context.Context) error { +func (rel *relayMiner) Stop(ctx context.Context) error { return rel.relayerProxy.Stop(ctx) } From a031b926835d5a782a4062a58e9b195f4eccc361 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:23:46 +0100 Subject: [PATCH 083/127] refactor: interrupt signal handling --- cmd/signals/on_exit.go | 23 +++++++++++++++++++++++ pkg/relayer/cmd/cmd.go | 18 ++++-------------- 2 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 cmd/signals/on_exit.go diff --git a/cmd/signals/on_exit.go b/cmd/signals/on_exit.go new file mode 100644 index 000000000..1106cce96 --- /dev/null +++ b/cmd/signals/on_exit.go @@ -0,0 +1,23 @@ +package signals + +import ( + "os" + "os/signal" +) + +// GoOnExitSignal calls the given callback when the process receives an interrupt +// or kill signal. +func GoOnExitSignal(onInterrupt func()) { + go func() { + // Set up sigCh to receive when this process receives an interrupt or + // kill signal. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt, os.Kill) + + // Block until we receive an interrupt or kill signal (OS-agnostic) + <-sigCh + + // Call the onInterrupt callback. + onInterrupt() + }() +} diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 912494c83..711cb08c4 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -5,8 +5,6 @@ import ( "fmt" "log" "net/url" - "os" - "os/signal" "cosmossdk.io/depinject" cosmosclient "github.com/cosmos/cosmos-sdk/client" @@ -14,6 +12,7 @@ import ( cosmostx "github.com/cosmos/cosmos-sdk/client/tx" "github.com/spf13/cobra" + "github.com/pokt-network/poktroll/cmd/signals" "github.com/pokt-network/poktroll/pkg/client/block" eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" "github.com/pokt-network/poktroll/pkg/client/supplier" @@ -68,6 +67,9 @@ func runRelayer(cmd *cobra.Command, _ []string) error { // Ensure context cancellation. defer cancelCtx() + // Handle interrupt and kill signals asynchronously. + signals.GoOnExitSignal(cancelCtx) + // Sets up the following dependencies: // Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, relayMiner dependencies. deps, err := setupRelayerDependencies(ctx, cmd) @@ -80,18 +82,6 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return err } - // Handle interrupts in a goroutine. - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - - // Block until we receive an interrupt or kill signal (OS-agnostic) - <-sigCh - - // Signal goroutines to stop - cancelCtx() - }() - // Start the relay miner log.Println("INFO: Starting relay miner...") if err := relayMiner.Start(ctx); err != nil { From c05ba2b292a54eb6c5919b5dc25730a081b386da Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:24:14 +0100 Subject: [PATCH 084/127] chore: improve comments --- pkg/relayer/cmd/cmd.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 711cb08c4..1431696c7 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -34,19 +34,20 @@ func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", Short: "Run a relay miner", - Long: `Run a relay miner`, - RunE: runRelayer, + // TODO_TECHDEBT: add a longer long description. + Long: `Run a relay miner`, + RunE: runRelayer, } cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - // TODO_TECHDEBT: integrate these cosmosflags with the client context (i.e. cosmosflags, config, viper, etc.) + // TODO_TECHDEBT: integrate these flags with the client context (i.e. cosmosflags, config, viper, etc.) // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE). // Will require more effort than currently justifiable. cmd.Flags().StringVar(&flagSigningKeyName, "signing-key", "", "Name of the key to sign transactions") cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to the SMT KV store") - // Communication cosmosflags - // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. Figure out + // Communication flags + // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. // Figure out what good defaults should be post alpha. cmd.Flags().StringVar(&flagSequencerNode, "sequencer-node", "explicitly omitting default", ": to sequencer node to submit txs") cmd.Flags().StringVar(&flagPocketNode, "pocket-node", "explicitly omitting default", ": to full pocket node for reading data and listening for on-chain events") @@ -71,7 +72,8 @@ func runRelayer(cmd *cobra.Command, _ []string) error { signals.GoOnExitSignal(cancelCtx) // Sets up the following dependencies: - // Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, relayMiner dependencies. + // Miner, EventsQueryClient, BlockClient, cosmosclient.Context, TxFactory, + // TxContext, TxClient, SupplierClient, RelayerProxy, RelayerSessionsManager. deps, err := setupRelayerDependencies(ctx, cmd) if err != nil { return err @@ -93,7 +95,8 @@ func runRelayer(cmd *cobra.Command, _ []string) error { } // setupRelayerDependencies sets up all the dependencies the relay miner needs to run: -// Miner, EventsQueryClient, BlockClient, TxClient, SupplierClient, RelayerProxy, relayMiner +// Miner, EventsQueryClient, BlockClient, cosmosclient.Context, TxFactory, TxContext, +// TxClient, SupplierClient, RelayerProxy, RelayerSessionsManager. func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, @@ -257,6 +260,8 @@ func supplyRelayerProxy( cmd *cobra.Command, ) (depinject.Config, error) { // TODO_TECHDEBT: this should be populated from some relayerProxy config. + // TODO_TECHDEBT(#179): this hostname should be updated to match that of the + // in-tilt anvil service. anvilURL, err := url.Parse("http://localhost:8547/") if err != nil { return nil, err From aeb4ee82c9f47953a14938fcd9f9539432d63abe Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:25:12 +0100 Subject: [PATCH 085/127] chore: improve comments --- Makefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 395ebc5e9..735cec340 100644 --- a/Makefile +++ b/Makefile @@ -362,22 +362,19 @@ supplier_stake: ## Stake tokens for the supplier specified (must specify the APP .PHONY: supplier1_stake supplier1_stake: ## Stake supplier1 (also staked in genesis) - # TODO_IMPROVE: consolidate supplier1 staking; why/where is supplier1 - # being staked other than this make target? - # Supplier1 seems to be staked at some point during localnet startup. - # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8548,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake supplier2_stake: ## Stake supplier2 - # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8548,svc2;http://localhost:8082" make supplier_stake .PHONY: supplier3_stake supplier3_stake: ## Stake supplier3 - # TODO_TECHDEBT: once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8548,svc3;http://localhost:8083" make supplier_stake From 6648a3846c7c97830b50fce3fa24786d1195b3a3 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:25:47 +0100 Subject: [PATCH 086/127] revert: tiltfile hot-reload dirs --- Tiltfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tiltfile b/Tiltfile index 8e9ab4e82..1333a406e 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,7 +2,7 @@ load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_resource", "helm_repo") # A list of directories where changes trigger a hot-reload of the sequencer -hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg/relayer"] +hot_reload_dirs = ["app", "cmd", "tools", "x"] # Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values localnet_config_path = "localnet_config.yaml" From e5a4d73abe4762061916de9750d5d32c1bfca445 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:25:54 +0100 Subject: [PATCH 087/127] refactor: re-consolidate client contexts --- pkg/client/tx/context.go | 9 ++------- pkg/relayer/cmd/cmd.go | 36 +++++++++++++++++++++++------------- pkg/relayer/interface.go | 6 ------ pkg/relayer/proxy/proxy.go | 8 ++------ 4 files changed, 27 insertions(+), 32 deletions(-) diff --git a/pkg/client/tx/context.go b/pkg/client/tx/context.go index 639e26e18..040731b46 100644 --- a/pkg/client/tx/context.go +++ b/pkg/client/tx/context.go @@ -12,7 +12,6 @@ import ( authclient "github.com/cosmos/cosmos-sdk/x/auth/client" "github.com/pokt-network/poktroll/pkg/client" - "github.com/pokt-network/poktroll/pkg/relayer" ) var _ client.TxContext = (*cosmosTxContext)(nil) @@ -34,19 +33,15 @@ type cosmosTxContext struct { func NewTxContext(deps depinject.Config) (client.TxContext, error) { txCtx := cosmosTxContext{} - var txClientCtx relayer.TxClientContext - if err := depinject.Inject( deps, - &txClientCtx, + &txCtx.clientCtx, &txCtx.txFactory, ); err != nil { return nil, err } - // TODO_CONSIDERATION: it might improve readability to use - // QueryClientContext on the struct. - txCtx.clientCtx = cosmosclient.Context(txClientCtx) + txCtx.clientCtx = cosmosclient.Context(txCtx.clientCtx) return txCtx, nil } diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 1431696c7..8180820f0 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -124,8 +124,11 @@ func setupRelayerDependencies( return nil, err } - // Has no dependencies. - deps, err = supplyTxClientCtxAndTxFactory(deps, cmd) + // Has no dependencies + deps, err = supplyClientContext(deps, cmd) + + // Depends on clientCtx. + deps, err = supplyTxFactory(deps, cmd) if err != nil { return nil, err } @@ -202,21 +205,36 @@ func supplyBlockClient( return depinject.Configs(deps, depinject.Supply(blockClient)), nil } -func supplyTxClientCtxAndTxFactory( +func supplyClientContext( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { + // NB: The implementation of #GetClientTxContext() is identical to that of + // #GetClientQueryContext(); either is sufficient for usage in both querying + // and transaction building and broadcasting. clientCtx, err := cosmosclient.GetClientTxContext(cmd) if err != nil { return nil, err } + deps = depinject.Configs(deps, depinject.Supply(clientCtx)) + return deps, nil +} + +func supplyTxFactory( + deps depinject.Config, + cmd *cobra.Command, +) (depinject.Config, error) { + var clientCtx cosmosclient.Context + if err := depinject.Inject(deps, &clientCtx); err != nil { + return nil, err + } + clientFactory, err := cosmostx.NewFactoryCLI(clientCtx, cmd.Flags()) if err != nil { return nil, err } - txClientCtx := relayer.TxClientContext(clientCtx) - return depinject.Configs(deps, depinject.Supply(txClientCtx, clientFactory)), nil + return depinject.Configs(deps, depinject.Supply(clientFactory)), nil } func supplyTxClient( @@ -271,14 +289,6 @@ func supplyRelayerProxy( "anvil": *anvilURL, } - clientCtx, err := cosmosclient.GetClientQueryContext(cmd) - if err != nil { - return nil, err - } - - queryClientCtx := relayer.QueryClientContext(clientCtx) - deps = depinject.Configs(deps, depinject.Supply(queryClientCtx)) - relayerProxy, err := proxy.NewRelayerProxy( deps, proxy.WithSigningKeyName(flagSigningKeyName), diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 3153ac9b7..539361f2d 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -3,7 +3,6 @@ package relayer import ( "context" - "github.com/cosmos/cosmos-sdk/client" "github.com/pokt-network/smt" "github.com/pokt-network/poktroll/pkg/observable" @@ -131,8 +130,3 @@ type SessionTree interface { // submitted on-chain and the servicer has confirmed that it has been rewarded. Delete() error } - -// TODO_IN_THIS_COMMIT: comment.. these exist to differentiate in -// depinject; therefore, can't be aliases. -type TxClientContext client.Context -type QueryClientContext client.Context diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index fff7af361..242ca0230 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -97,19 +97,15 @@ func NewRelayerProxy( ) (relayer.RelayerProxy, error) { rp := &relayerProxy{} - var queryClientCtx relayer.QueryClientContext - if err := depinject.Inject( deps, - &queryClientCtx, + &rp.clientCtx, &rp.blockClient, ); err != nil { return nil, err } - // TODO_CONSIDERATION: it might improve readability to use - // QueryClientContext on the struct. - rp.clientCtx = sdkclient.Context(queryClientCtx) + rp.clientCtx = sdkclient.Context(rp.clientCtx) servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() From 95fc559d516e954223d2c4d6179812f198d86b80 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:28:56 +0100 Subject: [PATCH 088/127] fix: typo --- pkg/appgateserver/jsonrpc.go | 6 +++--- pkg/client/tx/client.go | 2 +- pkg/client/tx/errors.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 5c778cde8..96115f6bf 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -81,12 +81,12 @@ func (app *appGateServer) handleJSONRPCRelay( // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := cdc.Marshal(relayRequest) if err != nil { - return ErrAppGateHandleRelay.Wrapf("marshalling relay request", err) + return ErrAppGateHandleRelay.Wrapf("marshaling relay request", err) } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) var relayReq types.RelayRequest if err := relayReq.Unmarshal(relayRequestBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("unmarshalling relay response", err) + return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response", err) } // Create the HTTP request to send the request to the relayer. @@ -113,7 +113,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} if err := relayResponse.Unmarshal(relayResponseBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("unmarshalling relay response", err) + return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response", err) } // Verify the response signature. We use the supplier address that we got from diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index a02b32542..aa2561832 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -517,7 +517,7 @@ func (tClient *txClient) txEventFromEventBz( return either.Error[*TxEvent](ErrUnmarshalTx.Wrapf("%s", err)), true } - // For successful unmarshalling, return the TxEvent. + // For successful unmarshaling, return the TxEvent. return either.Success(txEvt), false } diff --git a/pkg/client/tx/errors.go b/pkg/client/tx/errors.go index 1e43f1d05..328d7a51d 100644 --- a/pkg/client/tx/errors.go +++ b/pkg/client/tx/errors.go @@ -32,7 +32,7 @@ var ( // byte data isn't recognized as a valid transaction event representation. ErrNonTxEventBytes = errorsmod.Register(codespace, 9, "attempted to deserialize non-tx event bytes") - // ErrUnmarshalTx signals a failure in the unmarshalling process of a transaction. + // ErrUnmarshalTx signals a failure in the unmarshaling process of a transaction. // This error is triggered when the system encounters issues translating a set of // bytes into the corresponding Tx structure or object. ErrUnmarshalTx = errorsmod.Register(codespace, 10, "failed to unmarshal tx") From c3898e0dda4c25f875b009f26bb55a4b99f5aa44 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:29:04 +0100 Subject: [PATCH 089/127] chore: remove todo --- pkg/relayer/cmd/cmd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 8180820f0..01ef698ea 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -177,7 +177,6 @@ func supplyEventsQueryClient(deps depinject.Config, pocketNodeWebsocketURL strin return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil } -// TODO_IN_THIS_COMMIT: move func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { pocketNodeURLStr, err := cmd.Flags().GetString(cosmosflags.FlagNode) if err != nil { From c61abaf193ded9da5f4a9d53caf8e58603d756a5 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:29:37 +0100 Subject: [PATCH 090/127] chore: add todo comment --- config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.yml b/config.yml index 27af27ff8..789f712ad 100644 --- a/config.yml +++ b/config.yml @@ -100,6 +100,7 @@ genesis: - endpoints: - configs: [] rpc_type: JSON_RPC + # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to it instead of `localhost`. url: http://localhost:8548 service: id: anvil From 839a3dc384507b08938710685fc8f4faa593d170 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:32:23 +0100 Subject: [PATCH 091/127] revert: comment change --- pkg/relayer/relayminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/relayminer.go b/pkg/relayer/relayminer.go index 81ff82256..46ac9d5d8 100644 --- a/pkg/relayer/relayminer.go +++ b/pkg/relayer/relayminer.go @@ -8,7 +8,7 @@ import ( ) // relayMiner is the main struct that encapsulates the relayer's responsibilities (i.e. Relay Mining). -// It starts and stops the RelayerProxy and provide the served relays observable to them miner. +// It starts and stops the RelayerProxy and provide the served relays observable to the miner. type relayMiner struct { relayerProxy RelayerProxy miner Miner From ff8fc6ab33ce1671a2a3aadc65cece5998dbbebc Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:39:50 +0100 Subject: [PATCH 092/127] fix: error format strings --- pkg/appgateserver/jsonrpc.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 96115f6bf..bfb6670ce 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -25,7 +25,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Read the request body bytes. payloadBz, err := io.ReadAll(request.Body) if err != nil { - return ErrAppGateHandleRelay.Wrapf("reading relay request body", err) + return ErrAppGateHandleRelay.Wrapf("reading relay request body: %s", err) } log.Printf("DEBUG: relay request body: %s", string(payloadBz)) @@ -62,31 +62,31 @@ func (app *appGateServer) handleJSONRPCRelay( // Get the application's signer. signer, err := app.getRingSingerForAppAddress(ctx, appAddress) if err != nil { - return ErrAppGateHandleRelay.Wrapf("getting signer", err) + return ErrAppGateHandleRelay.Wrapf("getting signer: %s", err) } // Hash and sign the request's signable bytes. signableBz, err := relayRequest.GetSignableBytes() if err != nil { - return ErrAppGateHandleRelay.Wrapf("getting signable bytes", err) + return ErrAppGateHandleRelay.Wrapf("getting signable bytes: %s", err) } hash := crypto.Sha256(signableBz) signature, err := signer.Sign(hash) if err != nil { - return ErrAppGateHandleRelay.Wrapf("signing relay", err) + return ErrAppGateHandleRelay.Wrapf("signing relay: %s", err) } relayRequest.Meta.Signature = signature // Marshal the relay request to bytes and create a reader to be used as an HTTP request body. relayRequestBz, err := cdc.Marshal(relayRequest) if err != nil { - return ErrAppGateHandleRelay.Wrapf("marshaling relay request", err) + return ErrAppGateHandleRelay.Wrapf("marshaling relay request: %s", err) } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) var relayReq types.RelayRequest if err := relayReq.Unmarshal(relayRequestBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response", err) + return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response: %s", err) } // Create the HTTP request to send the request to the relayer. @@ -101,7 +101,7 @@ func (app *appGateServer) handleJSONRPCRelay( log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { - return ErrAppGateHandleRelay.Wrapf("sending relay request", err) + return ErrAppGateHandleRelay.Wrapf("sending relay request: %s", err) } // Read the response body bytes. @@ -113,7 +113,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Unmarshal the response bytes into a RelayResponse. relayResponse := &types.RelayResponse{} if err := relayResponse.Unmarshal(relayResponseBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response", err) + return ErrAppGateHandleRelay.Wrapf("unmarshaling relay response: %s", err) } // Verify the response signature. We use the supplier address that we got from @@ -124,7 +124,7 @@ func (app *appGateServer) handleJSONRPCRelay( // failed responses. if err := app.verifyResponse(ctx, supplierAddress, relayResponse); err != nil { // TODO_DISCUSS: should this be its own error type and asserted against in tests? - return ErrAppGateHandleRelay.Wrapf("verifying relay response signature", err) + return ErrAppGateHandleRelay.Wrapf("verifying relay response signature: %s", err) } // Marshal the response payload to bytes to be sent back to the application. @@ -136,7 +136,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Reply with the RelayResponse payload. log.Printf("DEBUG: Writing relay response payload: %s", string(relayResponsePayloadBz)) if _, err := writer.Write(relayResponsePayloadBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("writing relay response payload", err) + return ErrAppGateHandleRelay.Wrapf("writing relay response payload: %s", err) } return nil From 66bfba50d2baf2868a0356a7b65aa7bba3bba02c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:40:00 +0100 Subject: [PATCH 093/127] chore: remove comment --- pkg/relayer/cmd/cmd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 01ef698ea..994bdb03d 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -56,7 +56,6 @@ func RelayerCmd() *cobra.Command { // Set --node flag to the --sequencer-node for the client context err := cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", flagSequencerNode)) if err != nil { - //return nil, err panic(err) } From 95a6dc936bed906faa07d966ed3dc743e0dc0249 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 13:43:14 +0100 Subject: [PATCH 094/127] fix: error format strings --- pkg/appgateserver/jsonrpc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index bfb6670ce..860bf774e 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -107,7 +107,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Read the response body bytes. relayResponseBz, err := io.ReadAll(relayHTTPResponse.Body) if err != nil { - return ErrAppGateHandleRelay.Wrapf("reading relay response body", err) + return ErrAppGateHandleRelay.Wrapf("reading relay response body: %s", err) } // Unmarshal the response bytes into a RelayResponse. @@ -130,7 +130,7 @@ func (app *appGateServer) handleJSONRPCRelay( // Marshal the response payload to bytes to be sent back to the application. relayResponsePayloadBz, err := cdc.MarshalJSON(relayResponse.GetJsonRpcPayload()) if err != nil { - return ErrAppGateHandleRelay.Wrapf("unmarshallig relay response", err) + return ErrAppGateHandleRelay.Wrapf("unmarshallig relay response: %s", err) } // Reply with the RelayResponse payload. From 65ff2e4fa0a3dffcd590723560f333845d67e5a4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 14:39:35 +0100 Subject: [PATCH 095/127] chore: add `-features-path` flag to cucumber tests --- e2e/tests/init_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 4532596d9..31c7cba40 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -3,7 +3,9 @@ package e2e import ( + "flag" "fmt" + "log" "regexp" "strconv" "strings" @@ -31,12 +33,21 @@ var ( accNameToAppMap = make(map[string]apptypes.Application) accNameToSupplierMap = make(map[string]sharedtypes.Supplier) - keyRingFlag = "--keyring-backend=test" + featuresPathFlag string + keyRingFlag = "--keyring-backend=test" ) func init() { addrRe = regexp.MustCompile(`address:\s+(\S+)\s+name:\s+(\S+)`) amountRe = regexp.MustCompile(`amount:\s+"(.+?)"\s+denom:\s+upokt`) + + flag.StringVar(&featuresPathFlag, "features-path", "*.feature", "Specifies glob paths for the runner to look up .feature files") +} + +func TestMain(m *testing.M) { + flag.Parse() + log.Printf("features path: %s", featuresPathFlag) + m.Run() } type suite struct { @@ -58,7 +69,7 @@ func (s *suite) Before() { // TestFeatures runs the e2e tests specified in any .features files in this directory // * This test suite assumes that a LocalNet is running func TestFeatures(t *testing.T) { - gocuke.NewRunner(t, &suite{}).Path("*.feature").Run() + gocuke.NewRunner(t, &suite{}).Path(featuresPathFlag).Run() } func (s *suite) TheUserHasThePocketdBinaryInstalled() { From 5ec4514ddcd3eb9518884f32f78758118bfbc2c3 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 14:45:11 +0100 Subject: [PATCH 096/127] fix: set the relayminer URL in the curl cmd --- e2e/tests/init_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 31c7cba40..0051c5c67 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -254,7 +254,11 @@ func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName st } func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName, supplierName, serviceId, requestData string) { - res, err := s.pocketd.RunCurl("", serviceId, requestData) + // TODO_TECHDEBT(#179): Once relayminer and appgateserver are running in tilt, + // use their respective in-tilt hostnames and run E2E tests in tilt. This + // should match the on-chain advertised endpoint for the service with the + // given serviceId. + res, err := s.pocketd.RunCurl("http://localhost:8545", serviceId, requestData) if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } From 1160c0aac5e13c653ad0c470651dd6d1697db976 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 14:45:32 +0100 Subject: [PATCH 097/127] chore: remove redundant `-X` curl arg (says curl) --- e2e/tests/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 2aee2eba7..4345ec45d 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -109,7 +109,7 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { // runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided func (p *pocketdBin) runCurlPostCmd(rpcUrl string, service string, data string, args ...string) (*commandResult, error) { - base := []string{"-v", "-X", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), fmt.Sprintf("%s/%s", rpcUrl, service)} + base := []string{"-v", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), fmt.Sprintf("%s/%s", rpcUrl, service)} args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command cmd := exec.Command("curl", args...) From b7a66de4d210962ec1393d34b2a56e861e6e8afd Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 15:04:28 +0100 Subject: [PATCH 098/127] squash: fix relayminer url: reword: s/relayminer/appgateserver/ --- e2e/tests/init_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 0051c5c67..67a358c62 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -258,7 +258,7 @@ func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName // use their respective in-tilt hostnames and run E2E tests in tilt. This // should match the on-chain advertised endpoint for the service with the // given serviceId. - res, err := s.pocketd.RunCurl("http://localhost:8545", serviceId, requestData) + res, err := s.pocketd.RunCurl("http://localhost:42069", serviceId, requestData) if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } From 49ba4161845ac15596d716708005b44429976649 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 17:06:18 +0100 Subject: [PATCH 099/127] chore: improve error messaging --- pkg/appgateserver/relay_verifier.go | 11 ++++++++++- pkg/relayer/proxy/relay_verifier.go | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 4f86e1cdd..17dc1edca 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -26,7 +26,16 @@ func (app *appGateServer) verifyResponse( // Extract the supplier's signature if relayResponse.Meta == nil { - return ErrAppGateEmptyRelayResponse + payload := relayResponse.GetPayload() + payloadBz := make([]byte, payload.Size()) + if _, err := payload.MarshalTo(payloadBz); err != nil { + return ErrAppGateEmptyRelayResponseSignature.Wrapf( + "unable to unmarshal relay response payload: %s", err, + ) + } + return ErrAppGateEmptyRelayResponseSignature.Wrapf( + "response payload: %s", relayResponse.Payload, + ) } supplierSignature := relayResponse.Meta.SupplierSignature diff --git a/pkg/relayer/proxy/relay_verifier.go b/pkg/relayer/proxy/relay_verifier.go index ba8bf7e32..38da8c911 100644 --- a/pkg/relayer/proxy/relay_verifier.go +++ b/pkg/relayer/proxy/relay_verifier.go @@ -97,7 +97,7 @@ func (rp *relayerProxy) VerifyRelayRequest( // matches the relayRequest sessionId. // TODO_INVESTIGATE: Revisit the assumptions above at some point in the future, but good enough for now. if session.SessionId != relayRequest.Meta.SessionHeader.SessionId { - return ErrRelayerProxyInvalidSession + return ErrRelayerProxyInvalidSession.Wrapf("%+v", session) } // Check if the relayRequest is allowed to be served by the relayer proxy. From 7e400e462a8b05b82cf8bc3073f379850d753df8 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 17:07:12 +0100 Subject: [PATCH 100/127] fix: curl invocation --- e2e/tests/node.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/tests/node.go b/e2e/tests/node.go index 4345ec45d..afd42a8c8 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -109,7 +109,9 @@ func (p *pocketdBin) runPocketCmd(args ...string) (*commandResult, error) { // runCurlPostCmd is a helper to run a command using the local pocketd binary with the flags provided func (p *pocketdBin) runCurlPostCmd(rpcUrl string, service string, data string, args ...string) (*commandResult, error) { - base := []string{"-v", "POST", "-H", "'Content-Type: application/json'", "--data", fmt.Sprintf("'%s'", data), fmt.Sprintf("%s/%s", rpcUrl, service)} + dataStr := fmt.Sprintf("%s", data) + urlStr := fmt.Sprintf("%s/%s", rpcUrl, service) + base := []string{"-v", "POST", "-H", "Content-Type: application/json", "--data", dataStr, urlStr} args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command cmd := exec.Command("curl", args...) From b52f046966cb78f1be7d66f5686d51a7d2d9bad1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 17:07:37 +0100 Subject: [PATCH 101/127] test: implement step definition to assert agains relay response --- e2e/tests/init_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 67a358c62..c7ef89d2e 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -262,11 +262,22 @@ func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } - fmt.Println("OLSH Res", res.Stdout) + + relayKey := getRelayKey(appName, supplierName) + s.scenarioState[relayKey] = res.Stdout +} + +// TODO_IN_THIS_COMMIT: move +func getRelayKey(appName, supplierName string) string { + return fmt.Sprintf("%s/%s", appName, supplierName) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { - // TODO(#126, @Olshansk): Implement this step + relayKey := getRelayKey(appName, supplierName) + stdout, ok := s.scenarioState[relayKey] + + require.Truef(s, ok, "no relay response found for %s", relayKey) + require.Contains(s, stdout, `"result":"0x`) } func (s *suite) getStakedAmount(actorType, accName string) (bool, int) { From 7bc6a260a3dcf84801bf79739ec6c1ebb035bbfc Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 14 Nov 2023 17:07:58 +0100 Subject: [PATCH 102/127] chore: improve error name & messaging --- pkg/appgateserver/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index eecf99f44..551dc6e66 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -10,6 +10,6 @@ var ( ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 4, "missing application address") ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 5, "missing app client signing information") ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 6, "missing app client listening endpoint") - ErrAppGateEmptyRelayResponse = sdkerrors.Register(codespace, 7, "empty relay response") + ErrAppGateEmptyRelayResponseSignature = sdkerrors.Register(codespace, 7, "empty relay response signature") ErrAppGateHandleRelay = sdkerrors.Register(codespace, 8, "internal error handling relay request") ) From 700fd06c9f18b0571c5646e9e117dbaa7d4bf524 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 14 Nov 2023 12:33:08 -0800 Subject: [PATCH 103/127] Self review --- e2e/tests/init_test.go | 3 ++- e2e/tests/relay.feature | 6 ------ pkg/appgateserver/jsonrpc.go | 12 +----------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 4532596d9..454238264 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -247,7 +247,8 @@ func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } - fmt.Println("OLSH Res", res.Stdout) + // TODO(#184): store & use the result of res + fmt.Println(res) } func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature index f5b6af236..484172092 100644 --- a/e2e/tests/relay.feature +++ b/e2e/tests/relay.feature @@ -2,12 +2,6 @@ Feature: Relay Namespace Scenario: App can send relay to Supplier Given the user has the pocketd binary installed - - # TEMP: DELETE - When the user sends "1000" uPOKT from account "app2" to account "app1" - When the user sends "1000" uPOKT from account "app2" to account "supplier1" - # TEMP: DELETE - And the application "app1" is staked for service "anvil" And the supplier "supplier1" is staked for service "anvil" And the session for application "app1" and service "anvil" contains the supplier "supplier1" diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 0e3f9fe16..80784fb44 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -3,7 +3,6 @@ package appgateserver import ( "bytes" "context" - "fmt" "io" "log" "net/http" @@ -79,7 +78,6 @@ func (app *appGateServer) handleJSONRPCRelay( return err } relayRequestReader := io.NopCloser(bytes.NewReader(relayRequestBz)) - // relayRequestReader := io.NopCloser(bytes.NewReader(payloadBz)) // Create the HTTP request to send the request to the relayer. relayHTTPRequest := &http.Request{ @@ -89,15 +87,7 @@ func (app *appGateServer) handleJSONRPCRelay( Body: relayRequestReader, } - // TODO: relayminer is currently named relayers - // application (localhost) - // -> appgate (localhost:42069); configured by the gateway/application **off-chain** - // -> relayminer (supplierURL); advertised **on-chain** - // -> anvil (localhost:8547); configured **behind-the-scenes**; chains.json (v0); currently hard-coded - - // Perform the HTTP request to the relayer. - log.Printf("DEBUG: Sending raw payload to signed relay request to %s", supplierUrl) - fmt.Printf("\n~~~~ OLSH %+v \n~~~~\n", relayHTTPRequest) + log.Printf("DEBUG: Sending signed relay request to %s", supplierUrl) relayHTTPResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { return err From 122bec0d4c4baa61a1371526c81e2ccaaea3268e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 08:55:28 +0100 Subject: [PATCH 104/127] fixup: merge upstream --- pkg/appgateserver/jsonrpc.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/appgateserver/jsonrpc.go b/pkg/appgateserver/jsonrpc.go index 1d70706f6..915167dc8 100644 --- a/pkg/appgateserver/jsonrpc.go +++ b/pkg/appgateserver/jsonrpc.go @@ -3,7 +3,6 @@ package appgateserver import ( "bytes" "context" - "fmt" "io" "log" "net/http" From 8c1f5b9d849bece0c195e77714cdd161939b610f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 08:55:40 +0100 Subject: [PATCH 105/127] chore: review feedback improvements --- e2e/tests/init_test.go | 35 ++++++++++++++++------------- pkg/appgateserver/errors.go | 5 +++-- pkg/appgateserver/relay_verifier.go | 4 ++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index c7ef89d2e..88a8afff4 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -24,6 +24,12 @@ import ( suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" ) +// TODO_TECHDEBT(#179): Once relayminer and appgateserver are running in tilt, +// use their respective in-tilt hostnames and run E2E tests in tilt. This +// should match the on-chain advertised endpoint for the service with the +// given serviceId. +const localnetAppGateServerUrl = "http://localhost:42069" + var ( addrRe *regexp.Regexp amountRe *regexp.Regexp @@ -33,7 +39,7 @@ var ( accNameToAppMap = make(map[string]apptypes.Application) accNameToSupplierMap = make(map[string]sharedtypes.Supplier) - featuresPathFlag string + flagFeaturesPath string keyRingFlag = "--keyring-backend=test" ) @@ -41,12 +47,12 @@ func init() { addrRe = regexp.MustCompile(`address:\s+(\S+)\s+name:\s+(\S+)`) amountRe = regexp.MustCompile(`amount:\s+"(.+?)"\s+denom:\s+upokt`) - flag.StringVar(&featuresPathFlag, "features-path", "*.feature", "Specifies glob paths for the runner to look up .feature files") + flag.StringVar(&flagFeaturesPath, "features-path", "*.feature", "Specifies glob paths for the runner to look up .feature files") } func TestMain(m *testing.M) { flag.Parse() - log.Printf("features path: %s", featuresPathFlag) + log.Printf("features path: %s", flagFeaturesPath) m.Run() } @@ -69,7 +75,7 @@ func (s *suite) Before() { // TestFeatures runs the e2e tests specified in any .features files in this directory // * This test suite assumes that a LocalNet is running func TestFeatures(t *testing.T) { - gocuke.NewRunner(t, &suite{}).Path(featuresPathFlag).Run() + gocuke.NewRunner(t, &suite{}).Path(flagFeaturesPath).Run() } func (s *suite) TheUserHasThePocketdBinaryInstalled() { @@ -254,26 +260,17 @@ func (s *suite) TheSessionForApplicationAndServiceContainsTheSupplier(appName st } func (s *suite) TheApplicationSendsTheSupplierARequestForServiceWithData(appName, supplierName, serviceId, requestData string) { - // TODO_TECHDEBT(#179): Once relayminer and appgateserver are running in tilt, - // use their respective in-tilt hostnames and run E2E tests in tilt. This - // should match the on-chain advertised endpoint for the service with the - // given serviceId. - res, err := s.pocketd.RunCurl("http://localhost:42069", serviceId, requestData) + res, err := s.pocketd.RunCurl(localnetAppGateServerUrl, serviceId, requestData) if err != nil { s.Fatalf("error sending relay request from app %s to supplier %s for service %s: %v", appName, supplierName, serviceId, err) } - relayKey := getRelayKey(appName, supplierName) + relayKey := relayReferenceKey(appName, supplierName) s.scenarioState[relayKey] = res.Stdout } -// TODO_IN_THIS_COMMIT: move -func getRelayKey(appName, supplierName string) string { - return fmt.Sprintf("%s/%s", appName, supplierName) -} - func (s *suite) TheApplicationReceivesASuccessfulRelayResponseSignedBy(appName string, supplierName string) { - relayKey := getRelayKey(appName, supplierName) + relayKey := relayReferenceKey(appName, supplierName) stdout, ok := s.scenarioState[relayKey] require.Truef(s, ok, "no relay response found for %s", relayKey) @@ -388,3 +385,9 @@ func (s *suite) getAccBalance(accName string) int { require.NoError(s, err) return found } + +// TODO_IMPROVE: use `sessionId` and `supplierName` since those are the two values +// used to create the primary composite key on-chain to uniquely distinguish relays. +func relayReferenceKey(appName, supplierName string) string { + return fmt.Sprintf("%s/%s", appName, supplierName) +} diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go index 551dc6e66..c07a7843f 100644 --- a/pkg/appgateserver/errors.go +++ b/pkg/appgateserver/errors.go @@ -10,6 +10,7 @@ var ( ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 4, "missing application address") ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 5, "missing app client signing information") ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 6, "missing app client listening endpoint") - ErrAppGateEmptyRelayResponseSignature = sdkerrors.Register(codespace, 7, "empty relay response signature") - ErrAppGateHandleRelay = sdkerrors.Register(codespace, 8, "internal error handling relay request") + ErrAppGateEmptyRelayResponseMeta = sdkerrors.Register(codespace, 7, "empty relay response metadata") + ErrAppGateEmptyRelayResponseSignature = sdkerrors.Register(codespace, 8, "empty relay response signature") + ErrAppGateHandleRelay = sdkerrors.Register(codespace, 9, "internal error handling relay request") ) diff --git a/pkg/appgateserver/relay_verifier.go b/pkg/appgateserver/relay_verifier.go index 17dc1edca..68e4cc418 100644 --- a/pkg/appgateserver/relay_verifier.go +++ b/pkg/appgateserver/relay_verifier.go @@ -29,8 +29,8 @@ func (app *appGateServer) verifyResponse( payload := relayResponse.GetPayload() payloadBz := make([]byte, payload.Size()) if _, err := payload.MarshalTo(payloadBz); err != nil { - return ErrAppGateEmptyRelayResponseSignature.Wrapf( - "unable to unmarshal relay response payload: %s", err, + return ErrAppGateEmptyRelayResponseMeta.Wrapf( + "unable to marshal relay response payload: %s", err, ) } return ErrAppGateEmptyRelayResponseSignature.Wrapf( From 5bfe151fd89fbdb20b87b07aea854f8018ba13f7 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 10:01:27 +0100 Subject: [PATCH 106/127] chore: update anvil service port in make targets --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 735cec340..5db5ec974 100644 --- a/Makefile +++ b/Makefile @@ -364,19 +364,19 @@ supplier_stake: ## Stake tokens for the supplier specified (must specify the APP supplier1_stake: ## Stake supplier1 (also staked in genesis) # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). - SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8548,svc1;http://localhost:8081" make supplier_stake + SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8545,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake supplier2_stake: ## Stake supplier2 # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). - SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8548,svc2;http://localhost:8082" make supplier_stake + SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8545,svc2;http://localhost:8082" make supplier_stake .PHONY: supplier3_stake supplier3_stake: ## Stake supplier3 # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). - SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8548,svc3;http://localhost:8083" make supplier_stake + SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8545,svc3;http://localhost:8083" make supplier_stake .PHONY: supplier_unstake supplier_unstake: ## Unstake an supplier (must specify the SUPPLIER env var) From 86255b3ebd90854a719cd9165ebc42c5f76cf5b5 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 10:14:12 +0100 Subject: [PATCH 107/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/cmd/cmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 994bdb03d..db9a46389 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -42,10 +42,10 @@ func RelayerCmd() *cobra.Command { cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") // TODO_TECHDEBT: integrate these flags with the client context (i.e. cosmosflags, config, viper, etc.) - // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE). - // Will require more effort than currently justifiable. + // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE) and requires more effort than currently justifiable. cmd.Flags().StringVar(&flagSigningKeyName, "signing-key", "", "Name of the key to sign transactions") - cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to the SMT KV store") + // TODO_TECHDEBT(#137): This, alongside other flags, should be part of a config file suppliers provide. + cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to where the data backing SMT KV store exists on disk") // Communication flags // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. // Figure out what good defaults should be post alpha. From 0aaddc84fa36acd3964907154065fe2f963e2703 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 10:49:29 +0100 Subject: [PATCH 108/127] refactor: relayminer depinject helpers & godoc comments on all constructors --- pkg/client/block/client.go | 3 + pkg/client/events_query/client.go | 5 + pkg/client/tx/client.go | 9 ++ pkg/client/tx/context.go | 6 +- pkg/relayer/cmd/cmd.go | 188 +++++++++++++++++------------- pkg/relayer/proxy/proxy.go | 14 ++- pkg/relayer/session/session.go | 7 ++ 7 files changed, 149 insertions(+), 83 deletions(-) diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index 375171d28..7ded373f2 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -75,6 +75,9 @@ type eventBytesToBlockMapFn = func( ) (client.Block, bool) // NewBlockClient creates a new block client from the given dependencies and cometWebsocketURL. +// +// Required dependencies: +// - client.EventsQueryClient func NewBlockClient( ctx context.Context, deps depinject.Config, diff --git a/pkg/client/events_query/client.go b/pkg/client/events_query/client.go index 88ace493f..34a7d35d2 100644 --- a/pkg/client/events_query/client.go +++ b/pkg/client/events_query/client.go @@ -62,6 +62,11 @@ func (ebc *eventsBytesAndConn) Close() { _ = ebc.conn.Close() } +// NewEventsQueryClient returns a new events query client which is used to +// subscribe to on-chain events matching the given query. +// +// Available options: +// - WithDialer func NewEventsQueryClient(cometWebsocketURL string, opts ...client.EventsQueryClientOption) client.EventsQueryClient { evtClient := &eventsQueryClient{ cometWebsocketURL: cometWebsocketURL, diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index aa2561832..39f5208e0 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -105,6 +105,15 @@ type TxEvent struct { // validateConfigAndSetDefaults method. // 5. Subscribes the client to its own transactions. This step might be // reconsidered for relocation to a potential Start() method in the future. +// +// Required dependencies: +// - client.TxContext +// - client.EventsQueryClient +// - client.BlockClient +// +// Available options: +// - WithSigningKeyName +// - WithCommitTimeoutHeightOffset func NewTxClient( ctx context.Context, deps depinject.Config, diff --git a/pkg/client/tx/context.go b/pkg/client/tx/context.go index 040731b46..01cf76382 100644 --- a/pkg/client/tx/context.go +++ b/pkg/client/tx/context.go @@ -30,6 +30,10 @@ type cosmosTxContext struct { // NewTxContext initializes a new cosmosTxContext with the given dependencies. // It uses depinject to populate its members and returns a client.TxContext // interface type. +// +// Required dependencies: +// - cosmosclient.Context +// - cosmostx.Factory func NewTxContext(deps depinject.Config) (client.TxContext, error) { txCtx := cosmosTxContext{} @@ -41,8 +45,6 @@ func NewTxContext(deps depinject.Config) (client.TxContext, error) { return nil, err } - txCtx.clientCtx = cosmosclient.Context(txCtx.clientCtx) - return txCtx, nil } diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index db9a46389..a13f41b4d 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -30,6 +30,12 @@ var ( flagPocketNode string ) +type supplierFn func( + context.Context, + depinject.Config, + *cobra.Command, +) (depinject.Config, error) + func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", @@ -44,7 +50,7 @@ func RelayerCmd() *cobra.Command { // TODO_TECHDEBT: integrate these flags with the client context (i.e. cosmosflags, config, viper, etc.) // This is simpler to do with server-side configs (see rootCmd#PersistentPreRunE) and requires more effort than currently justifiable. cmd.Flags().StringVar(&flagSigningKeyName, "signing-key", "", "Name of the key to sign transactions") - // TODO_TECHDEBT(#137): This, alongside other flags, should be part of a config file suppliers provide. + // TODO_TECHDEBT(#137): This, alongside other flags, should be part of a config file suppliers provide. cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to where the data backing SMT KV store exists on disk") // Communication flags // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. @@ -93,90 +99,45 @@ func runRelayer(cmd *cobra.Command, _ []string) error { return nil } -// setupRelayerDependencies sets up all the dependencies the relay miner needs to run: +// setupRelayerDependencies sets up all the dependencies the relay miner needs +// to run by building the dependency tree from the leaves up, incrementally +// supplying each component to an accumulating depinject.Config: // Miner, EventsQueryClient, BlockClient, cosmosclient.Context, TxFactory, TxContext, // TxClient, SupplierClient, RelayerProxy, RelayerSessionsManager. func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, ) (deps depinject.Config, err error) { - // Initizlize deps to with empty depinject config. - deps, err = supplyMiner(depinject.Configs()) + pocketNodeWebsocketUrl, err := getPocketNodeWebsocketUrl(cmd) if err != nil { return nil, err } - rpcQueryURL, err := getPocketNodeWebsocketURL(cmd) - if err != nil { - return nil, err + supplierFuncs := []supplierFn{ + newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf + newSupplyBlockClientFn(pocketNodeWebsocketUrl), + supplyMiner, // leaf + supplyClientContext, // leaf + supplyTxFactory, + supplyTxContext, + supplyTxClient, + supplySupplierClient, + supplyRelayerProxy, + supplyRelayerSessionsManager, } - // Has no dependencies. - deps, err = supplyEventsQueryClient(deps, rpcQueryURL) - if err != nil { - return nil, err - } - - // Depends on EventsQueryClient. - deps, err = supplyBlockClient(ctx, deps, rpcQueryURL) - if err != nil { - return nil, err - } - - // Has no dependencies - deps, err = supplyClientContext(deps, cmd) - - // Depends on clientCtx. - deps, err = supplyTxFactory(deps, cmd) - if err != nil { - return nil, err - } - - // Depends on clientCtx, txFactory, EventsQueryClient, & BlockClient. - deps, err = supplyTxClient(ctx, deps) - if err != nil { - return nil, err - } - - // Depends on txClient & EventsQueryClient. - deps, err = supplySupplierClient(deps) - if err != nil { - return nil, err - } - - // Depends on clientCtx & BlockClient. - deps, err = supplyRelayerProxy(deps, cmd) - if err != nil { - return nil, err - } - - // Depends on BlockClient & SupplierClient. - deps, err = supplyRelayerSessionsManager(ctx, deps) - if err != nil { - return nil, err + // Initialize deps to with empty depinject config. + deps = depinject.Configs() + for _, supplyFn := range supplierFuncs { + deps, err = supplyFn(ctx, deps, cmd) } return deps, nil } -func supplyMiner( - deps depinject.Config, -) (depinject.Config, error) { - mnr, err := miner.NewMiner() - if err != nil { - return nil, err - } - - return depinject.Configs(deps, depinject.Supply(mnr)), nil -} - -func supplyEventsQueryClient(deps depinject.Config, pocketNodeWebsocketURL string) (depinject.Config, error) { - eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketURL) - - return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil -} - -func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { +// getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to +// connect to for subscribing to on-chain events. +func getPocketNodeWebsocketUrl(cmd *cobra.Command) (string, error) { pocketNodeURLStr, err := cmd.Flags().GetString(cosmosflags.FlagNode) if err != nil { return "", err @@ -190,20 +151,62 @@ func getPocketNodeWebsocketURL(cmd *cobra.Command) (string, error) { return fmt.Sprintf("ws://%s/websocket", pocketNodeURL.Host), nil } -func supplyBlockClient( - ctx context.Context, +// newSupplyEventsQueryClientFn constructs an EventsQueryClient instance and returns +// a new depinject.Config which is supplied with the given deps and the new +// EventsQueryClient. +func newSupplyEventsQueryClientFn( + pocketNodeWebsocketUrl string, +) supplierFn { + return func( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketUrl) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil + } +} + +// newSupplyBlockClientFn returns a function with constructs a BlockClient instance +// with the given nodeURL and returns a new +// depinject.Config which is supplied with the given deps and the new +// BlockClient. +func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) supplierFn { + return func( + ctx context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + blockClient, err := block.NewBlockClient(ctx, deps, pocketNodeWebsocketUrl) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(blockClient)), nil + } +} + +// supplyMiner constructs a Miner instance and returns a new depinject.Config +// which is supplied with the given deps and the new Miner. +func supplyMiner( + _ context.Context, deps depinject.Config, - nodeURL string, + _ *cobra.Command, ) (depinject.Config, error) { - blockClient, err := block.NewBlockClient(ctx, deps, nodeURL) + mnr, err := miner.NewMiner() if err != nil { return nil, err } - return depinject.Configs(deps, depinject.Supply(blockClient)), nil + return depinject.Configs(deps, depinject.Supply(mnr)), nil } +// supplyClientContext constructs a cosmosclient.Context instance and returns a +// new depinject.Config which is supplied with the given deps and the new +// cosmosclient.Context. func supplyClientContext( + _ context.Context, deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { @@ -218,7 +221,11 @@ func supplyClientContext( return deps, nil } +// supplyTxFactory constructs a cosmostx.Factory instance and returns a new +// depinject.Config which is supplied with the given deps and the new +// cosmostx.Factory. func supplyTxFactory( + _ context.Context, deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { @@ -235,16 +242,26 @@ func supplyTxFactory( return depinject.Configs(deps, depinject.Supply(clientFactory)), nil } -func supplyTxClient( - ctx context.Context, +func supplyTxContext( + _ context.Context, deps depinject.Config, + _ *cobra.Command, ) (depinject.Config, error) { txContext, err := tx.NewTxContext(deps) if err != nil { return nil, err } - deps = depinject.Configs(deps, depinject.Supply(txContext)) + return depinject.Configs(deps, depinject.Supply(txContext)), nil +} + +// supplyTxClient constructs a TxClient instance and returns a new +// depinject.Config which is supplied with the given deps and the new TxClient. +func supplyTxClient( + ctx context.Context, + deps depinject.Config, + _ *cobra.Command, +) (depinject.Config, error) { txClient, err := tx.NewTxClient( ctx, deps, @@ -259,7 +276,14 @@ func supplyTxClient( return depinject.Configs(deps, depinject.Supply(txClient)), nil } -func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { +// supplySupplierClient constructs a SupplierClient instance and returns a new +// depinject.Config which is supplied with the given deps and the new +// SupplierClient. +func supplySupplierClient( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, +) (depinject.Config, error) { supplierClient, err := supplier.NewSupplierClient( deps, supplier.WithSigningKeyName(flagSigningKeyName), @@ -271,9 +295,13 @@ func supplySupplierClient(deps depinject.Config) (depinject.Config, error) { return depinject.Configs(deps, depinject.Supply(supplierClient)), nil } +// supplyRelayerProxy constructs a RelayerProxy instance and returns a new +// depinject.Config which is supplied with the given deps and the new +// RelayerProxy. func supplyRelayerProxy( + _ context.Context, deps depinject.Config, - cmd *cobra.Command, + _ *cobra.Command, ) (depinject.Config, error) { // TODO_TECHDEBT: this should be populated from some relayerProxy config. // TODO_TECHDEBT(#179): this hostname should be updated to match that of the @@ -299,9 +327,13 @@ func supplyRelayerProxy( return depinject.Configs(deps, depinject.Supply(relayerProxy)), nil } +// supplyRelayerSessionsManager constructs a RelayerSessionsManager instance +// and returns a new depinject.Config which is supplied with the given deps and +// the new RelayerSessionsManager. func supplyRelayerSessionsManager( ctx context.Context, deps depinject.Config, + _ *cobra.Command, ) (depinject.Config, error) { relayerSessionsManager, err := session.NewRelayerSessions( ctx, deps, diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 242ca0230..b97ef4695 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -7,7 +7,7 @@ import ( "cosmossdk.io/depinject" ringtypes "github.com/athanorlabs/go-dleq/types" - sdkclient "github.com/cosmos/cosmos-sdk/client" + cosmosclient "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/crypto/keyring" accounttypes "github.com/cosmos/cosmos-sdk/x/auth/types" "golang.org/x/sync/errgroup" @@ -83,7 +83,7 @@ type relayerProxy struct { ringCacheMutex *sync.RWMutex // clientCtx is the Cosmos' client context used to build the needed query clients and unmarshal their replies. - clientCtx sdkclient.Context + clientCtx cosmosclient.Context // supplierAddress is the address of the supplier that the relayer proxy is running for. supplierAddress string @@ -91,6 +91,14 @@ type relayerProxy struct { // NewRelayerProxy creates a new relayer proxy with the given dependencies or returns // an error if the dependencies fail to resolve or the options are invalid. +// +// Required dependencies: +// - cosmosclient.Context +// - client.BlockClient +// +// Available options: +// - WithSigningKeyName +// - WithProxiedServicesEndpoints func NewRelayerProxy( deps depinject.Config, opts ...relayer.RelayerProxyOption, @@ -105,7 +113,7 @@ func NewRelayerProxy( return nil, err } - rp.clientCtx = sdkclient.Context(rp.clientCtx) + rp.clientCtx = cosmosclient.Context(rp.clientCtx) servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 0a0f66aff..9f4441f82 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -45,6 +45,13 @@ type relayerSessionsManager struct { } // NewRelayerSessions creates a new relayerSessions. +// +// Required dependencies: +// - client.BlockClient +// - client.SupplierClient +// +// Available options: +// - WithStoresDirectory func NewRelayerSessions( ctx context.Context, deps depinject.Config, From 6ad6abebeaf7575a6f2dcd766376b4922c6af8f2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 12:43:34 +0100 Subject: [PATCH 109/127] =?UTF-8?q?refactor:=20separate=20tx=20and=20query?= =?UTF-8?q?=20client=20contexts=20=F0=9F=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/client/tx/context.go | 9 +-- pkg/relayer/cmd/cmd.go | 100 ++++++++++++++++++++--------- pkg/relayer/interface.go | 13 ++++ pkg/relayer/proxy/proxy.go | 13 ++-- pkg/relayer/session/sessiontree.go | 2 +- 5 files changed, 95 insertions(+), 42 deletions(-) diff --git a/pkg/client/tx/context.go b/pkg/client/tx/context.go index 01cf76382..2bfa0da5f 100644 --- a/pkg/client/tx/context.go +++ b/pkg/client/tx/context.go @@ -12,6 +12,7 @@ import ( authclient "github.com/cosmos/cosmos-sdk/x/auth/client" "github.com/pokt-network/poktroll/pkg/client" + "github.com/pokt-network/poktroll/pkg/relayer" ) var _ client.TxContext = (*cosmosTxContext)(nil) @@ -21,7 +22,7 @@ var _ client.TxContext = (*cosmosTxContext)(nil) type cosmosTxContext struct { // Holds cosmos-sdk client context. // (see: https://pkg.go.dev/github.com/cosmos/cosmos-sdk@v0.47.5/client#Context) - clientCtx cosmosclient.Context + clientCtx relayer.TxClientContext // Holds the cosmos-sdk transaction factory. // (see: https://pkg.go.dev/github.com/cosmos/cosmos-sdk@v0.47.5/client/tx#Factory) txFactory cosmostx.Factory @@ -64,7 +65,7 @@ func (txCtx cosmosTxContext) SignTx( ) error { return authclient.SignTx( txCtx.txFactory, - txCtx.clientCtx, + cosmosclient.Context(txCtx.clientCtx), signingKeyName, txBuilder, offline, overwriteSig, @@ -84,8 +85,8 @@ func (txCtx cosmosTxContext) EncodeTx(txBuilder cosmosclient.TxBuilder) ([]byte, // BroadcastTx broadcasts the given transaction to the network, blocking until the check-tx // ABCI operation completes and returns a TxResponse of the transaction status at that point in time. func (txCtx cosmosTxContext) BroadcastTx(txBytes []byte) (*cosmostypes.TxResponse, error) { - return txCtx.clientCtx.BroadcastTxAsync(txBytes) - //return txCtx.clientCtx.BroadcastTxSync(txBytes) + clientCtx := cosmosclient.Context(txCtx.clientCtx) + return clientCtx.BroadcastTxAsync(txBytes) } // QueryTx queries the transaction based on its hash and optionally provides proof diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index a13f41b4d..17561d149 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -23,11 +23,13 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/session" ) +const omittedDefaultFlagValue = "explicitly omitting default" + var ( - flagSigningKeyName string - flagSmtStorePath string - flagSequencerNode string - flagPocketNode string + flagSigningKeyName string + flagSmtStorePath string + flagSequencerNodeUrl string + flagPocketNodeUrl string ) type supplierFn func( @@ -55,15 +57,9 @@ func RelayerCmd() *cobra.Command { // Communication flags // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. // Figure out what good defaults should be post alpha. - cmd.Flags().StringVar(&flagSequencerNode, "sequencer-node", "explicitly omitting default", ": to sequencer node to submit txs") - cmd.Flags().StringVar(&flagPocketNode, "pocket-node", "explicitly omitting default", ": to full pocket node for reading data and listening for on-chain events") - cmd.Flags().String(cosmosflags.FlagNode, "explicitly omitting default", "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") - - // Set --node flag to the --sequencer-node for the client context - err := cmd.Flags().Set(cosmosflags.FlagNode, fmt.Sprintf("tcp://%s", flagSequencerNode)) - if err != nil { - panic(err) - } + cmd.Flags().StringVar(&flagSequencerNodeUrl, "sequencer-node", "explicitly omitting default", "tcp://: to sequencer node to submit txs") + cmd.Flags().StringVar(&flagPocketNodeUrl, "pocket-node", omittedDefaultFlagValue, "tcp://: to full pocket node for reading data and listening for on-chain events") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") return cmd } @@ -108,7 +104,7 @@ func setupRelayerDependencies( ctx context.Context, cmd *cobra.Command, ) (deps depinject.Config, err error) { - pocketNodeWebsocketUrl, err := getPocketNodeWebsocketUrl(cmd) + pocketNodeWebsocketUrl, err := getPocketNodeWebsocketUrl() if err != nil { return nil, err } @@ -116,8 +112,9 @@ func setupRelayerDependencies( supplierFuncs := []supplierFn{ newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf newSupplyBlockClientFn(pocketNodeWebsocketUrl), - supplyMiner, // leaf - supplyClientContext, // leaf + supplyMiner, // leaf + supplyQueryClientContext, // leaf + supplyTxClientContext, // leaf supplyTxFactory, supplyTxContext, supplyTxClient, @@ -130,6 +127,9 @@ func setupRelayerDependencies( deps = depinject.Configs() for _, supplyFn := range supplierFuncs { deps, err = supplyFn(ctx, deps, cmd) + if err != nil { + return nil, err + } } return deps, nil @@ -137,13 +137,12 @@ func setupRelayerDependencies( // getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to // connect to for subscribing to on-chain events. -func getPocketNodeWebsocketUrl(cmd *cobra.Command) (string, error) { - pocketNodeURLStr, err := cmd.Flags().GetString(cosmosflags.FlagNode) - if err != nil { - return "", err +func getPocketNodeWebsocketUrl() (string, error) { + if flagPocketNodeUrl == omittedDefaultFlagValue { + return "", fmt.Errorf("--pocket-node flag is required") } - pocketNodeURL, err := url.Parse(pocketNodeURLStr) + pocketNodeURL, err := url.Parse(flagPocketNodeUrl) if err != nil { return "", err } @@ -202,22 +201,62 @@ func supplyMiner( return depinject.Configs(deps, depinject.Supply(mnr)), nil } -// supplyClientContext constructs a cosmosclient.Context instance and returns a +func supplyQueryClientContext( + _ context.Context, + deps depinject.Config, + cmd *cobra.Command, +) (depinject.Config, error) { + // Set --node flag to the --pocket-node for the client context + // This flag is read by cosmosclient.GetClientQueryContext. + err := cmd.Flags().Set(cosmosflags.FlagNode, flagPocketNodeUrl) + if err != nil { + return nil, err + } + + // NB: Currently, the implementations of GetClientTxContext() and + // GetClientQueryContext() are identical, allowing for their interchangeable + // use in both querying and transaction operations. However, in order to support + // independent configuration of client contexts for distinct querying and + // transacting purposes. E.g.: transactions are dispatched to the sequencer + // while queries are handled by a trusted full-node. + queryClientCtx, err := cosmosclient.GetClientQueryContext(cmd) + if err != nil { + return nil, err + } + deps = depinject.Configs(deps, depinject.Supply( + relayer.QueryClientContext(queryClientCtx), + )) + return deps, nil +} + +// supplyTxClientContext constructs a cosmosclient.Context instance and returns a // new depinject.Config which is supplied with the given deps and the new // cosmosclient.Context. -func supplyClientContext( +func supplyTxClientContext( _ context.Context, deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - // NB: The implementation of #GetClientTxContext() is identical to that of - // #GetClientQueryContext(); either is sufficient for usage in both querying - // and transaction building and broadcasting. - clientCtx, err := cosmosclient.GetClientTxContext(cmd) + // Set --node flag to the --sequencer-node for this client context. + // This flag is read by cosmosclient.GetClientTxContext. + err := cmd.Flags().Set(cosmosflags.FlagNode, flagSequencerNodeUrl) + if err != nil { + return nil, err + } + + // NB: Currently, the implementations of GetClientTxContext() and + // GetClientQueryContext() are identical, allowing for their interchangeable + // use in both querying and transaction operations. However, in order to support + // independent configuration of client contexts for distinct querying and + // transacting purposes. E.g.: transactions are dispatched to the sequencer + // while queries are handled by a trusted full-node. + txClientCtx, err := cosmosclient.GetClientTxContext(cmd) if err != nil { return nil, err } - deps = depinject.Configs(deps, depinject.Supply(clientCtx)) + deps = depinject.Configs(deps, depinject.Supply( + relayer.TxClientContext(txClientCtx), + )) return deps, nil } @@ -229,11 +268,12 @@ func supplyTxFactory( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - var clientCtx cosmosclient.Context - if err := depinject.Inject(deps, &clientCtx); err != nil { + var txClientCtx relayer.TxClientContext + if err := depinject.Inject(deps, &txClientCtx); err != nil { return nil, err } + clientCtx := cosmosclient.Context(txClientCtx) clientFactory, err := cosmostx.NewFactoryCLI(clientCtx, cmd.Flags()) if err != nil { return nil, err diff --git a/pkg/relayer/interface.go b/pkg/relayer/interface.go index 539361f2d..d58d8149a 100644 --- a/pkg/relayer/interface.go +++ b/pkg/relayer/interface.go @@ -3,6 +3,7 @@ package relayer import ( "context" + "github.com/cosmos/cosmos-sdk/client" "github.com/pokt-network/smt" "github.com/pokt-network/poktroll/pkg/observable" @@ -11,6 +12,18 @@ import ( sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) +// TxClientContext is used to distinguish a cosmosclient.Context intended for use +// in transactions from others. +// This type is intentionally not an alias in order to make this distinction clear +// to the dependency injector +type TxClientContext client.Context + +// QueryClientContext is used to distinguish a cosmosclient.Context intended for use +// in queries from others. +// This type is intentionally not an alias in order to make this distinction clear +// to the dependency injector +type QueryClientContext client.Context + // Miner is responsible for observing servedRelayObs, hashing and checking the // difficulty of each, finally publishing those with sufficient difficulty to // minedRelayObs as they are applicable for relay volume. diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index b97ef4695..5e168712f 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -83,7 +83,7 @@ type relayerProxy struct { ringCacheMutex *sync.RWMutex // clientCtx is the Cosmos' client context used to build the needed query clients and unmarshal their replies. - clientCtx cosmosclient.Context + clientCtx relayer.QueryClientContext // supplierAddress is the address of the supplier that the relayer proxy is running for. supplierAddress string @@ -113,16 +113,15 @@ func NewRelayerProxy( return nil, err } - rp.clientCtx = cosmosclient.Context(rp.clientCtx) - + clientCtx := cosmosclient.Context(rp.clientCtx) servedRelays, servedRelaysProducer := channel.NewObservable[*types.Relay]() rp.servedRelays = servedRelays rp.servedRelaysPublishCh = servedRelaysProducer - rp.accountsQuerier = accounttypes.NewQueryClient(rp.clientCtx) - rp.supplierQuerier = suppliertypes.NewQueryClient(rp.clientCtx) - rp.sessionQuerier = sessiontypes.NewQueryClient(rp.clientCtx) - rp.applicationQuerier = apptypes.NewQueryClient(rp.clientCtx) + rp.accountsQuerier = accounttypes.NewQueryClient(clientCtx) + rp.supplierQuerier = suppliertypes.NewQueryClient(clientCtx) + rp.sessionQuerier = sessiontypes.NewQueryClient(clientCtx) + rp.applicationQuerier = apptypes.NewQueryClient(clientCtx) rp.keyring = rp.clientCtx.Keyring rp.ringCache = make(map[string][]ringtypes.Point) rp.ringCacheMutex = &sync.RWMutex{} diff --git a/pkg/relayer/session/sessiontree.go b/pkg/relayer/session/sessiontree.go index 2ae0cfb52..9ce6dbf32 100644 --- a/pkg/relayer/session/sessiontree.go +++ b/pkg/relayer/session/sessiontree.go @@ -68,7 +68,7 @@ func NewSessionTree( storePath := filepath.Join(storesDirectory, sessionHeader.SessionId) // Make sure storePath does not exist when creating a new SessionTree - if _, err := os.Stat(storePath); !os.IsNotExist(err) { + if _, err := os.Stat(storePath); err != nil && !os.IsNotExist(err) { return nil, ErrSessionTreeUndefinedStoresDirectory } From 88e046d74dcdea01b90c78807c08a7f48ab4b263 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 12:47:21 +0100 Subject: [PATCH 110/127] fix: sessiontree store path check --- pkg/relayer/session/sessiontree.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/relayer/session/sessiontree.go b/pkg/relayer/session/sessiontree.go index 9ce6dbf32..18f9c777a 100644 --- a/pkg/relayer/session/sessiontree.go +++ b/pkg/relayer/session/sessiontree.go @@ -68,8 +68,12 @@ func NewSessionTree( storePath := filepath.Join(storesDirectory, sessionHeader.SessionId) // Make sure storePath does not exist when creating a new SessionTree - if _, err := os.Stat(storePath); err != nil && !os.IsNotExist(err) { - return nil, ErrSessionTreeUndefinedStoresDirectory + if _, err := os.Stat(storePath); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + return nil, ErrSessionTreeStorePathExists } treeStore, err := smt.NewKVStore(storePath) From 9b836919617b6f0cc760278fda4d0447105c1987 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 12:52:26 +0100 Subject: [PATCH 111/127] fix: sessiontree store path check --- pkg/relayer/session/sessiontree.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/relayer/session/sessiontree.go b/pkg/relayer/session/sessiontree.go index 18f9c777a..f9f03eaa8 100644 --- a/pkg/relayer/session/sessiontree.go +++ b/pkg/relayer/session/sessiontree.go @@ -68,12 +68,8 @@ func NewSessionTree( storePath := filepath.Join(storesDirectory, sessionHeader.SessionId) // Make sure storePath does not exist when creating a new SessionTree - if _, err := os.Stat(storePath); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - - return nil, ErrSessionTreeStorePathExists + if _, err := os.Stat(storePath); err != nil && !os.IsNotExist(err) { + return nil, ErrSessionTreeStorePathExists.Wrapf("storePath: %q", storePath) } treeStore, err := smt.NewKVStore(storePath) From c865eb07dcc2847591b45bc0c7b74658b1998a74 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 12:55:13 +0100 Subject: [PATCH 112/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- pkg/relayer/cmd/cmd.go | 6 ++++-- pkg/relayer/proxy/proxy.go | 2 +- pkg/relayer/session/session.go | 2 +- proto/pocket/service/relay.proto | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 17561d149..e2264be26 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -343,14 +343,16 @@ func supplyRelayerProxy( deps depinject.Config, _ *cobra.Command, ) (depinject.Config, error) { - // TODO_TECHDEBT: this should be populated from some relayerProxy config. + // TODO_BLOCKER:(#137): This MUST be populated via the `relayer.json` config file // TODO_TECHDEBT(#179): this hostname should be updated to match that of the // in-tilt anvil service. - anvilURL, err := url.Parse("http://localhost:8547/") + proxyServiceURL, err := url.Parse("http://localhost:8547/") if err != nil { return nil, err } + // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented an a local LLM node + // is supported, this needs to be expanded such that a single relayer can proxy to multiple services at once. proxiedServiceEndpoints := map[string]url.URL{ "anvil": *anvilURL, } diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 5e168712f..28f92576c 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -137,7 +137,7 @@ func NewRelayerProxy( return rp, nil } -// Start concurrently starts all advertised relay servers and returns an error +// Start concurrently starts all advertised relay services and returns an error // if any of them errors. // This method IS BLOCKING until all RelayServers are stopped. func (rp *relayerProxy) Start(ctx context.Context) error { diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 9f4441f82..ef89e874c 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -91,7 +91,7 @@ func NewRelayerSessions( // The session trees are piped through a series of map operations which progress // them through the claim/proof lifecycle, broadcasting transactions to the // network as necessary. -// It IS NOT blocking as map operations run in their own goroutines. +// It IS NOT BLOCKING as map operations run in their own goroutines. func (rs *relayerSessionsManager) Start(ctx context.Context) { // Map eitherMinedRelays to a new observable of an error type which is // notified if an error was encountered while attempting to add the relay to diff --git a/proto/pocket/service/relay.proto b/proto/pocket/service/relay.proto index 968fd8dfe..dc269fad0 100644 --- a/proto/pocket/service/relay.proto +++ b/proto/pocket/service/relay.proto @@ -40,7 +40,7 @@ message JSONRPCRequestPayload { uint32 id = 1; // Identifier established by the Client to create context for the request. string jsonrpc = 2; // Version of JSON-RPC. Must be exactly "2.0". string method = 3; // Method being invoked on the server. - // TODO_RESEARCH: Find out why the params being a map causes errors + // TODO_TECHDEBT(#126): Make params a `oneof` of a list or map per the JSON-RPC specifications // should they be a list of maps? //map params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures repeated string params = 4; // Parameters for the method. https://www.jsonrpc.org/specification#parameter_structures From fe9db5822f2f70f7879f82a000c26c52145ef46d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 13:20:25 +0100 Subject: [PATCH 113/127] chore: review feedback improvements --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 5db5ec974..7a770d038 100644 --- a/Makefile +++ b/Makefile @@ -364,6 +364,7 @@ supplier_stake: ## Stake tokens for the supplier specified (must specify the APP supplier1_stake: ## Stake supplier1 (also staked in genesis) # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). + # TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8545,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake From 8d7e577b8c8937869e785188bd29c177bde83b92 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 13:20:37 +0100 Subject: [PATCH 114/127] chore: add long command description --- pkg/relayer/cmd/cmd.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index e2264be26..a4db8c1ba 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -43,7 +43,18 @@ func RelayerCmd() *cobra.Command { Use: "relayminer", Short: "Run a relay miner", // TODO_TECHDEBT: add a longer long description. - Long: `Run a relay miner`, + Long: `Run a relay miner. The relay miner process configures and starts +relay servers for each service the supplier actor identified by --signing-key is +staked for (configured on-chain). Relay requests received by the relay servers +are validated and proxied to their respective service endpoints. The responses +are then signed and sent back to the requesting application. + +For each successfully served relay, the miner will hash and compare its difficulty +against an on-chain threshold. If the difficulty is sufficient, it is applicable +to relay volume and therefore rewards. Such relays are inserted into and persisted +via an SMT KV store. The miner will monitor the current block height and periodically +submit claim and proof messages according to the protocol as sessions become eligable +for such operations.`, RunE: runRelayer, } @@ -351,10 +362,10 @@ func supplyRelayerProxy( return nil, err } - // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented an a local LLM node + // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented an a local LLM node // is supported, this needs to be expanded such that a single relayer can proxy to multiple services at once. proxiedServiceEndpoints := map[string]url.URL{ - "anvil": *anvilURL, + "anvil": *proxyServiceURL, } relayerProxy, err := proxy.NewRelayerProxy( From f9455135a8f672a58cfefba76b2571d6a05d7f81 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 15 Nov 2023 13:35:22 +0100 Subject: [PATCH 115/127] fix: supplier client test --- testutil/testclient/testtx/context.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testutil/testclient/testtx/context.go b/testutil/testclient/testtx/context.go index 35b3dfb71..60412baa7 100644 --- a/testutil/testclient/testtx/context.go +++ b/testutil/testclient/testtx/context.go @@ -17,6 +17,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/testutil/mockclient" "github.com/pokt-network/poktroll/pkg/client" @@ -264,7 +265,8 @@ func NewAnyTimesTxTxContext( require.NoError(t, err) require.NotEmpty(t, txFactory) - txCtxDeps := depinject.Supply(txFactory, clientCtx) + txClientCtx := relayer.TxClientContext(clientCtx) + txCtxDeps := depinject.Supply(txFactory, txClientCtx) txCtx, err := tx.NewTxContext(txCtxDeps) require.NoError(t, err) txCtxMock := mockclient.NewMockTxContext(ctrl) From 4ce831b7de798867f7964c1d353ae0b59ec7b421 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:40:09 +0000 Subject: [PATCH 116/127] chore: cleanup flags and dependencies for appgateserver cmd --- pkg/appgateserver/cmd/cmd.go | 137 ++++++++++++++++++++++++++--------- 1 file changed, 104 insertions(+), 33 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index a9ff2908f..3ab648a93 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -7,26 +7,33 @@ import ( "log" "net/http" "net/url" - "os" - "os/signal" "cosmossdk.io/depinject" cosmosclient "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/client/flags" + cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" "github.com/spf13/cobra" + "github.com/pokt-network/poktroll/cmd/signals" "github.com/pokt-network/poktroll/pkg/appgateserver" - blockclient "github.com/pokt-network/poktroll/pkg/client/block" + "github.com/pokt-network/poktroll/pkg/client/block" eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" ) +const omittedDefaultFlagValue = "explicitly omitting default" + var ( flagSigningKey string flagSelfSigning bool flagListeningEndpoint string - flagCometWebsocketUrl string + flagQueryNodeUrl string ) +type supplierFn func( + context.Context, + depinject.Config, + *cobra.Command, +) (depinject.Config, error) + func AppGateServerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "appgate-server", @@ -58,11 +65,11 @@ relays to the AppGate server and function as an Application, provided that: cmd.Flags().StringVar(&flagSigningKey, "signing-key", "", "The name of the key that will be used to sign relays") cmd.Flags().StringVar(&flagListeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the appgate server will listen on") - cmd.Flags().StringVar(&flagCometWebsocketUrl, "comet-websocket-url", "ws://localhost:36657/websocket", "The URL of the comet websocket endpoint to communicate with the pocket blockchain") cmd.Flags().BoolVar(&flagSelfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") + cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "The URL of the pocket node to query for on-chain data") - cmd.Flags().String(flags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - cmd.Flags().String(flags.FlagNode, "tcp://localhost:36657", "The URL of the comet tcp endpoint to communicate with the pocket blockchain") + cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "The URL of the comet tcp endpoint to communicate with the pocket blockchain") return cmd } @@ -72,18 +79,8 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { ctx, cancelCtx := context.WithCancel(cmd.Context()) defer cancelCtx() - // Handle interrupts in a goroutine. - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - - // Block until we receive an interrupt or kill signal (OS-agnostic) - <-sigCh - log.Println("INFO: Interrupt signal received, shutting down...") - - // Signal goroutines to stop - cancelCtx() - }() + // Handle interrupt and kill signals asynchronously. + signals.GoOnExitSignal(cancelCtx) // Parse the listening endpoint. listeningUrl, err := url.Parse(flagListeningEndpoint) @@ -92,7 +89,7 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { } // Setup the AppGate server dependencies. - appGateServerDeps, err := setupAppGateServerDependencies(cmd, ctx, flagCometWebsocketUrl) + appGateServerDeps, err := setupAppGateServerDependencies(ctx, cmd) if err != nil { return fmt.Errorf("failed to setup AppGate server dependencies: %w", err) } @@ -127,21 +124,95 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { return nil } -func setupAppGateServerDependencies(cmd *cobra.Command, ctx context.Context, cometWebsocketUrl string) (depinject.Config, error) { - // Retrieve the client context for the chain interactions. - clientCtx := cosmosclient.GetClientContextFromCmd(cmd) +func setupAppGateServerDependencies( + ctx context.Context, + cmd *cobra.Command, +) (depinject.Config, error) { + pocketNodeWebsocketUrl, err := getPocketNodeWebsocketUrl() + if err != nil { + return nil, err + } + + supplierFuncs := []supplierFn{ + newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), + newSupplyBlockClientFn(pocketNodeWebsocketUrl), + newSupplyClientContextFn(cmd), + } + + // Initialize deps to with empty depinject config. + deps := depinject.Configs() + for _, supplyFn := range supplierFuncs { + deps, err = supplyFn(ctx, deps, cmd) + if err != nil { + return nil, err + } + } + + return deps, nil +} - // Create the events client. - eventsQueryClient := eventsquery.NewEventsQueryClient(cometWebsocketUrl) +// getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to +// connect to for subscribing to on-chain events. +func getPocketNodeWebsocketUrl() (string, error) { + if flagQueryNodeUrl == omittedDefaultFlagValue { + return "", errors.New("missing required flag: --query-node") + } - // Create the block client. - log.Printf("INFO: Creating block client, using comet websocket URL: %s...", cometWebsocketUrl) - deps := depinject.Supply(eventsQueryClient) - blockClient, err := blockclient.NewBlockClient(ctx, deps, cometWebsocketUrl) + pocketNodeURL, err := url.Parse(flagQueryNodeUrl) if err != nil { - return nil, fmt.Errorf("failed to create block client: %w", err) + return "", err } - // Return the dependencie config. - return depinject.Supply(clientCtx, blockClient), nil + return fmt.Sprintf("ws://%s/websocket", pocketNodeURL.Host), nil +} + +// newSupplyEventsQueryClientFn constructs an EventsQueryClient instance and returns +// a new depinject.Config which is supplied with the given deps and the new +// EventsQueryClient. +func newSupplyEventsQueryClientFn( + pocketNodeWebsocketUrl string, +) supplierFn { + return func( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketUrl) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil + } +} + +// newSupplyBlockClientFn returns a function with constructs a BlockClient instance +// with the given nodeURL and returns a new +// depinject.Config which is supplied with the given deps and the new +// BlockClient. +func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) supplierFn { + return func( + ctx context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + blockClient, err := block.NewBlockClient(ctx, deps, pocketNodeWebsocketUrl) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(blockClient)), nil + } +} + +// newSupplyClientContextFn returns a function with constructs a ClientContext instance +// with the given cmd and returns a new depinject.Config which is supplied with +// the given deps and the new ClientContext. +func newSupplyClientContextFn(cmd *cobra.Command) supplierFn { + return func( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + clientCtx := cosmosclient.GetClientContextFromCmd(cmd) + + return depinject.Configs(deps, depinject.Supply(clientCtx)), nil + } } From d83c1e74b09535bb8125d21b24bb8ac7ee9f5efa Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:05:27 +0000 Subject: [PATCH 117/127] chore: move shared dependency setup logic to shared pkg --- pkg/appgateserver/cmd/cmd.go | 89 +++++++++++------------------------- pkg/deps/config/config.go | 73 +++++++++++++++++++++++++++++ pkg/relayer/cmd/cmd.go | 31 ++++--------- 3 files changed, 110 insertions(+), 83 deletions(-) create mode 100644 pkg/deps/config/config.go diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 3ab648a93..e539cf7bd 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -15,8 +15,7 @@ import ( "github.com/pokt-network/poktroll/cmd/signals" "github.com/pokt-network/poktroll/pkg/appgateserver" - "github.com/pokt-network/poktroll/pkg/client/block" - eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" + "github.com/pokt-network/poktroll/pkg/deps/config" ) const omittedDefaultFlagValue = "explicitly omitting default" @@ -28,12 +27,6 @@ var ( flagQueryNodeUrl string ) -type supplierFn func( - context.Context, - depinject.Config, - *cobra.Command, -) (depinject.Config, error) - func AppGateServerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "appgate-server", @@ -133,22 +126,13 @@ func setupAppGateServerDependencies( return nil, err } - supplierFuncs := []supplierFn{ - newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), - newSupplyBlockClientFn(pocketNodeWebsocketUrl), - newSupplyClientContextFn(cmd), - } - - // Initialize deps to with empty depinject config. - deps := depinject.Configs() - for _, supplyFn := range supplierFuncs { - deps, err = supplyFn(ctx, deps, cmd) - if err != nil { - return nil, err - } + supplierFuncs := []config.SupplierFn{ + config.NewSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), + config.NewSupplyBlockClientFn(pocketNodeWebsocketUrl), + newSupplyQueryClientContextFn(flagQueryNodeUrl), } - return deps, nil + return config.SupplyConfig(ctx, cmd, supplierFuncs) } // getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to @@ -166,53 +150,34 @@ func getPocketNodeWebsocketUrl() (string, error) { return fmt.Sprintf("ws://%s/websocket", pocketNodeURL.Host), nil } -// newSupplyEventsQueryClientFn constructs an EventsQueryClient instance and returns -// a new depinject.Config which is supplied with the given deps and the new -// EventsQueryClient. -func newSupplyEventsQueryClientFn( - pocketNodeWebsocketUrl string, -) supplierFn { +// newSupplyQueryClientContextFn returns a new depinject.Config which is supplied with +// the given deps and a new cosmos ClientCtx +func newSupplyQueryClientContextFn(pocketQueryClientUrl string) config.SupplierFn { return func( _ context.Context, deps depinject.Config, - _ *cobra.Command, - ) (depinject.Config, error) { - eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketUrl) - - return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil - } -} - -// newSupplyBlockClientFn returns a function with constructs a BlockClient instance -// with the given nodeURL and returns a new -// depinject.Config which is supplied with the given deps and the new -// BlockClient. -func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) supplierFn { - return func( - ctx context.Context, - deps depinject.Config, - _ *cobra.Command, + cmd *cobra.Command, ) (depinject.Config, error) { - blockClient, err := block.NewBlockClient(ctx, deps, pocketNodeWebsocketUrl) + // Set --node flag to the pocketQueryClientUrl for the client context + // This flag is read by cosmosclient.GetClientQueryContext. + err := cmd.Flags().Set(cosmosflags.FlagNode, pocketQueryClientUrl) if err != nil { return nil, err } - return depinject.Configs(deps, depinject.Supply(blockClient)), nil - } -} - -// newSupplyClientContextFn returns a function with constructs a ClientContext instance -// with the given cmd and returns a new depinject.Config which is supplied with -// the given deps and the new ClientContext. -func newSupplyClientContextFn(cmd *cobra.Command) supplierFn { - return func( - _ context.Context, - deps depinject.Config, - _ *cobra.Command, - ) (depinject.Config, error) { - clientCtx := cosmosclient.GetClientContextFromCmd(cmd) - - return depinject.Configs(deps, depinject.Supply(clientCtx)), nil + // NB: Currently, the implementations of GetClientTxContext() and + // GetClientQueryContext() are identical, allowing for their interchangeable + // use in both querying and transaction operations. However, in order to support + // independent configuration of client contexts for distinct querying and + // transacting purposes. E.g.: transactions are dispatched to the sequencer + // while queries are handled by a trusted full-node. + queryClientCtx, err := cosmosclient.GetClientQueryContext(cmd) + if err != nil { + return nil, err + } + deps = depinject.Configs(deps, depinject.Supply( + queryClientCtx, + )) + return deps, nil } } diff --git a/pkg/deps/config/config.go b/pkg/deps/config/config.go new file mode 100644 index 000000000..9621fa1de --- /dev/null +++ b/pkg/deps/config/config.go @@ -0,0 +1,73 @@ +package config + +import ( + "context" + + "cosmossdk.io/depinject" + "github.com/spf13/cobra" + + "github.com/pokt-network/poktroll/pkg/client/block" + eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" +) + +// SupplierFn is a function that is used to supply a depinject config. +type SupplierFn func( + context.Context, + depinject.Config, + *cobra.Command, +) (depinject.Config, error) + +// SupplyConfig supplies a depinject config by calling each of the supplied +// supplier functions in order and passing the result of each supplier to the +// next supplier, chaining them together. +func SupplyConfig( + ctx context.Context, + cmd *cobra.Command, + suppliers []SupplierFn, +) (deps depinject.Config, err error) { + // Initialize deps to with empty depinject config. + deps = depinject.Configs() + for _, supplyFn := range suppliers { + deps, err = supplyFn(ctx, deps, cmd) + if err != nil { + return nil, err + } + } + return deps, nil +} + +// NewSupplyEventsQueryClientFn constructs an EventsQueryClient instance and returns +// a new depinject.Config which is supplied with the given deps and the new +// EventsQueryClient. +func NewSupplyEventsQueryClientFn( + pocketNodeWebsocketUrl string, +) SupplierFn { + return func( + _ context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketUrl) + + return depinject.Configs(deps, depinject.Supply(eventsQueryClient)), nil + } +} + +// NewSupplyBlockClientFn returns a function with constructs a BlockClient instance +// with the given nodeURL and returns a new +// depinject.Config which is supplied with the given deps and the new +// BlockClient. +func NewSupplyBlockClientFn(pocketNodeWebsocketUrl string) SupplierFn { + return func( + ctx context.Context, + deps depinject.Config, + _ *cobra.Command, + ) (depinject.Config, error) { + blockClient, err := block.NewBlockClient(ctx, deps, pocketNodeWebsocketUrl) + if err != nil { + return nil, err + } + + return depinject.Configs(deps, depinject.Supply(blockClient)), nil + } +} diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index a4db8c1ba..1f69b5397 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -17,6 +17,7 @@ import ( eventsquery "github.com/pokt-network/poktroll/pkg/client/events_query" "github.com/pokt-network/poktroll/pkg/client/supplier" "github.com/pokt-network/poktroll/pkg/client/tx" + "github.com/pokt-network/poktroll/pkg/deps/config" "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/pkg/relayer/miner" "github.com/pokt-network/poktroll/pkg/relayer/proxy" @@ -32,12 +33,6 @@ var ( flagPocketNodeUrl string ) -type supplierFn func( - context.Context, - depinject.Config, - *cobra.Command, -) (depinject.Config, error) - func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", @@ -120,9 +115,9 @@ func setupRelayerDependencies( return nil, err } - supplierFuncs := []supplierFn{ - newSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf - newSupplyBlockClientFn(pocketNodeWebsocketUrl), + supplierFuncs := []config.SupplierFn{ + config.NewSupplyEventsQueryClientFn(pocketNodeWebsocketUrl), // leaf + config.NewSupplyBlockClientFn(pocketNodeWebsocketUrl), supplyMiner, // leaf supplyQueryClientContext, // leaf supplyTxClientContext, // leaf @@ -134,16 +129,7 @@ func setupRelayerDependencies( supplyRelayerSessionsManager, } - // Initialize deps to with empty depinject config. - deps = depinject.Configs() - for _, supplyFn := range supplierFuncs { - deps, err = supplyFn(ctx, deps, cmd) - if err != nil { - return nil, err - } - } - - return deps, nil + return config.SupplyConfig(ctx, cmd, supplierFuncs) } // getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to @@ -166,7 +152,7 @@ func getPocketNodeWebsocketUrl() (string, error) { // EventsQueryClient. func newSupplyEventsQueryClientFn( pocketNodeWebsocketUrl string, -) supplierFn { +) config.SupplierFn { return func( _ context.Context, deps depinject.Config, @@ -182,7 +168,7 @@ func newSupplyEventsQueryClientFn( // with the given nodeURL and returns a new // depinject.Config which is supplied with the given deps and the new // BlockClient. -func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) supplierFn { +func newSupplyBlockClientFn(pocketNodeWebsocketUrl string) config.SupplierFn { return func( ctx context.Context, deps depinject.Config, @@ -212,6 +198,9 @@ func supplyMiner( return depinject.Configs(deps, depinject.Supply(mnr)), nil } +// supplyQueryClientContext returns a function with constructs a ClientContext +// instance with the given cmd and returns a new depinject.Config which is +// supplied with the given deps and the new ClientContext. func supplyQueryClientContext( _ context.Context, deps depinject.Config, From e58c8480c599295e30186f19457fff5dbfc32971 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:19:29 +0000 Subject: [PATCH 118/127] chore: update comment --- pkg/appgateserver/cmd/cmd.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index e539cf7bd..079f0a1a0 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -165,12 +165,7 @@ func newSupplyQueryClientContextFn(pocketQueryClientUrl string) config.SupplierF return nil, err } - // NB: Currently, the implementations of GetClientTxContext() and - // GetClientQueryContext() are identical, allowing for their interchangeable - // use in both querying and transaction operations. However, in order to support - // independent configuration of client contexts for distinct querying and - // transacting purposes. E.g.: transactions are dispatched to the sequencer - // while queries are handled by a trusted full-node. + // Get the client context from the command. queryClientCtx, err := cosmosclient.GetClientQueryContext(cmd) if err != nil { return nil, err From 75343bb3104bdf0ac7f68e0ffc87e58eba637540 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 12:05:10 -0800 Subject: [PATCH 119/127] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dc18d9d57..336be35aa 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ localnet_config.yaml release # SMT KVStore files +# TODO_TECHDEBT(#126, @red-0ne): Rename `smt` to `smt_stores` and make it configurable so it can be stored anywhere on this smt # Do not allow a multi-moduled projected From 1cc7085385d31058a3c18c2888c6822e96e946d1 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 12:12:03 -0800 Subject: [PATCH 120/127] Update OpenAPI spec --- .gitignore | 2 -- docs/static/openapi.yml | 30 ++++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 336be35aa..723084503 100644 --- a/.gitignore +++ b/.gitignore @@ -70,5 +70,3 @@ go.work.sum # TODO_IN_THIS_COMMIT: Why did we start generating .dot files? # **/*.dot - - diff --git a/docs/static/openapi.yml b/docs/static/openapi.yml index d82f777d7..f55a851f2 100644 --- a/docs/static/openapi.yml +++ b/docs/static/openapi.yml @@ -46480,7 +46480,7 @@ paths: service: title: >- The Service for which the application is - configured + configured for type: object properties: id: @@ -46660,7 +46660,9 @@ paths: type: object properties: service: - title: The Service for which the application is configured + title: >- + The Service for which the application is configured + for type: object properties: id: @@ -47176,7 +47178,7 @@ paths: service: title: >- The Service for which the application is - configured + configured for type: object properties: id: @@ -47243,7 +47245,7 @@ paths: service: title: >- The Service for which the supplier is - configured + configured for type: object properties: id: @@ -76787,7 +76789,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -76871,7 +76873,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -76965,7 +76967,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77016,7 +77018,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77284,7 +77286,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77348,7 +77350,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77538,7 +77540,7 @@ definitions: type: object properties: service: - title: The Service for which the application is configured + title: The Service for which the application is configured for type: object properties: id: @@ -77600,7 +77602,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77814,7 +77816,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: @@ -77943,7 +77945,7 @@ definitions: type: object properties: service: - title: The Service for which the supplier is configured + title: The Service for which the supplier is configured for type: object properties: id: From a04ebfc33d623eeb8f0cd7ee50a1b983d4e8e8e5 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 12:45:13 -0800 Subject: [PATCH 121/127] Updated comments for post 177+179 work for okdas --- .gitignore | 2 +- Makefile | 9 +++++---- config.yml | 2 +- pkg/appgateserver/cmd/cmd.go | 4 +++- pkg/relayer/cmd/cmd.go | 4 +++- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 723084503..8fe75f2aa 100644 --- a/.gitignore +++ b/.gitignore @@ -68,5 +68,5 @@ smt # Do not allow a multi-moduled projected go.work.sum -# TODO_IN_THIS_COMMIT: Why did we start generating .dot files? +# TODO_TECHDEBT: It seems that .dot files come and go so we need to figure out the root cause: https://github.com/pokt-network/poktroll/pull/177/files#r1392521547 # **/*.dot diff --git a/Makefile b/Makefile index 7a770d038..9200fb190 100644 --- a/Makefile +++ b/Makefile @@ -280,6 +280,7 @@ app_list: ## List all the staked applications app_stake: ## Stake tokens for the application specified (must specify the APP and SERVICES env vars) poktrolld --home=$(POKTROLLD_HOME) tx application stake-application 1000upokt $(SERVICES) --keyring-backend test --from $(APP) --node $(POCKET_NODE) +# TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper .PHONY: app1_stake app1_stake: ## Stake app1 (also staked in genesis) APP=app1 SERVICES=anvil,svc1,svc2 make app_stake @@ -360,22 +361,22 @@ supplier_list: ## List all the staked supplier supplier_stake: ## Stake tokens for the supplier specified (must specify the APP env var) poktrolld --home=$(POKTROLLD_HOME) tx supplier stake-supplier 1000upokt "$(SERVICES)" --keyring-backend test --from $(SUPPLIER) --node $(POCKET_NODE) +# TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper .PHONY: supplier1_stake supplier1_stake: ## Stake supplier1 (also staked in genesis) - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). - # TODO_IMPROVE(#180): Make sure genesis-staked actors are available via AccountKeeper SUPPLIER=supplier1 SERVICES="anvil;http://localhost:8545,svc1;http://localhost:8081" make supplier_stake .PHONY: supplier2_stake supplier2_stake: ## Stake supplier2 - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier2 SERVICES="anvil;http://localhost:8545,svc2;http://localhost:8082" make supplier_stake .PHONY: supplier3_stake supplier3_stake: ## Stake supplier3 - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to that service. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to that service. # I.e.: replace `localhost` with `relayminer` (or whatever the service's hostname is). SUPPLIER=supplier3 SERVICES="anvil;http://localhost:8545,svc3;http://localhost:8083" make supplier_stake diff --git a/config.yml b/config.yml index 789f712ad..90229f193 100644 --- a/config.yml +++ b/config.yml @@ -100,7 +100,7 @@ genesis: - endpoints: - configs: [] rpc_type: JSON_RPC - # TODO_TECHDEBT(#179): once `relayminer` service is added to tilt, this hostname should point to it instead of `localhost`. + # TODO_UPNEXT(@okdas): once `relayminer` service is added to tilt, this hostname should point to it instead of `localhost`. url: http://localhost:8548 service: id: anvil diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 079f0a1a0..d691403e8 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -24,7 +24,9 @@ var ( flagSigningKey string flagSelfSigning bool flagListeningEndpoint string - flagQueryNodeUrl string + // TODO_DISCUSS: Should we use `--node` for both querying and sending transactions, or have the respective + // `--network-node` for txs and `--query-node` for querying in the future? + flagQueryNodeUrl string ) func AppGateServerCmd() *cobra.Command { diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 1f69b5397..a41820a40 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -344,7 +344,7 @@ func supplyRelayerProxy( _ *cobra.Command, ) (depinject.Config, error) { // TODO_BLOCKER:(#137): This MUST be populated via the `relayer.json` config file - // TODO_TECHDEBT(#179): this hostname should be updated to match that of the + // TODO_UPNEXT(@okdas): this hostname should be updated to match that of the // in-tilt anvil service. proxyServiceURL, err := url.Parse("http://localhost:8547/") if err != nil { @@ -372,6 +372,8 @@ func supplyRelayerProxy( // supplyRelayerSessionsManager constructs a RelayerSessionsManager instance // and returns a new depinject.Config which is supplied with the given deps and // the new RelayerSessionsManager. +// See the comment next to `flagQueryNodeUrl` (if it still exists) on how/why +// we have multiple flags pointing to different node types. func supplyRelayerSessionsManager( ctx context.Context, deps depinject.Config, From 59f863c0c81573bd099121b380784be88dd368c0 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 12:58:32 -0800 Subject: [PATCH 122/127] Update pkg/relayer/cmd/cmd.go --- pkg/relayer/cmd/cmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index a41820a40..55da288c6 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -26,6 +26,7 @@ import ( const omittedDefaultFlagValue = "explicitly omitting default" +// TODO_CONSIDERATION: Consider moving all flags defined in `/pkg` to a `flags.go` file. var ( flagSigningKeyName string flagSmtStorePath string From 753c27ae97b8abf067e97f5698c18df3bc6d4e33 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 13:17:42 -0800 Subject: [PATCH 123/127] Update the names and references to queryNode/sequencerNode/fullNode etc --- pkg/appgateserver/cmd/cmd.go | 9 +++++---- pkg/relayer/cmd/cmd.go | 29 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index d691403e8..a2e71bed6 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -18,15 +18,14 @@ import ( "github.com/pokt-network/poktroll/pkg/deps/config" ) +// We're `explicitly omitting default` so that the appgateserver crashes if these aren't specified. const omittedDefaultFlagValue = "explicitly omitting default" var ( flagSigningKey string flagSelfSigning bool flagListeningEndpoint string - // TODO_DISCUSS: Should we use `--node` for both querying and sending transactions, or have the respective - // `--network-node` for txs and `--query-node` for querying in the future? - flagQueryNodeUrl string + flagQueryNodeUrl string ) func AppGateServerCmd() *cobra.Command { @@ -58,11 +57,13 @@ relays to the AppGate server and function as an Application, provided that: RunE: runAppGateServer, } + // Custom flags cmd.Flags().StringVar(&flagSigningKey, "signing-key", "", "The name of the key that will be used to sign relays") cmd.Flags().StringVar(&flagListeningEndpoint, "listening-endpoint", "http://localhost:42069", "The host and port that the appgate server will listen on") cmd.Flags().BoolVar(&flagSelfSigning, "self-signing", false, "Whether the server should sign all incoming requests with its own ring (for applications)") - cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "The URL of the pocket node to query for on-chain data") + cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "tcp://: to a full pocket node for reading data and listening for on-chain events") + // Cosmos flags cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "The URL of the comet tcp endpoint to communicate with the pocket blockchain") diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 55da288c6..e5a5d1729 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -24,14 +24,15 @@ import ( "github.com/pokt-network/poktroll/pkg/relayer/session" ) +// We're `explicitly omitting default` so the relayer crashes if these aren't specified. const omittedDefaultFlagValue = "explicitly omitting default" // TODO_CONSIDERATION: Consider moving all flags defined in `/pkg` to a `flags.go` file. var ( - flagSigningKeyName string - flagSmtStorePath string - flagSequencerNodeUrl string - flagPocketNodeUrl string + flagSigningKeyName string + flagSmtStorePath string + flagNetworkNodeUrl string + flagQueryNodeUrl string ) func RelayerCmd() *cobra.Command { @@ -62,10 +63,8 @@ for such operations.`, // TODO_TECHDEBT(#137): This, alongside other flags, should be part of a config file suppliers provide. cmd.Flags().StringVar(&flagSmtStorePath, "smt-store", "smt", "Path to where the data backing SMT KV store exists on disk") // Communication flags - // TODO_TECHDEBT: We're using `explicitly omitting default` so the relayer crashes if these aren't specified. - // Figure out what good defaults should be post alpha. - cmd.Flags().StringVar(&flagSequencerNodeUrl, "sequencer-node", "explicitly omitting default", "tcp://: to sequencer node to submit txs") - cmd.Flags().StringVar(&flagPocketNodeUrl, "pocket-node", omittedDefaultFlagValue, "tcp://: to full pocket node for reading data and listening for on-chain events") + cmd.Flags().StringVar(&flagNetworkNodeUrl, "network-node", omittedDefaultFlagValue, "tcp://: to a pocket node that gossips transactions throughout the network (may or may not be the sequencer") + cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "tcp://: to a full pocket node for reading data and listening for on-chain events") cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") return cmd @@ -136,11 +135,11 @@ func setupRelayerDependencies( // getPocketNodeWebsocketUrl returns the websocket URL of the Pocket Node to // connect to for subscribing to on-chain events. func getPocketNodeWebsocketUrl() (string, error) { - if flagPocketNodeUrl == omittedDefaultFlagValue { - return "", fmt.Errorf("--pocket-node flag is required") + if flagQueryNodeUrl == omittedDefaultFlagValue { + return "", fmt.Errorf("--query-node flag is required") } - pocketNodeURL, err := url.Parse(flagPocketNodeUrl) + pocketNodeURL, err := url.Parse(flagQueryNodeUrl) if err != nil { return "", err } @@ -207,9 +206,9 @@ func supplyQueryClientContext( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - // Set --node flag to the --pocket-node for the client context + // Set --node flag to the --query-node for the client context // This flag is read by cosmosclient.GetClientQueryContext. - err := cmd.Flags().Set(cosmosflags.FlagNode, flagPocketNodeUrl) + err := cmd.Flags().Set(cosmosflags.FlagNode, flagQueryNodeUrl) if err != nil { return nil, err } @@ -238,9 +237,9 @@ func supplyTxClientContext( deps depinject.Config, cmd *cobra.Command, ) (depinject.Config, error) { - // Set --node flag to the --sequencer-node for this client context. + // Set --node flag to the --network-node for this client context. // This flag is read by cosmosclient.GetClientTxContext. - err := cmd.Flags().Set(cosmosflags.FlagNode, flagSequencerNodeUrl) + err := cmd.Flags().Set(cosmosflags.FlagNode, flagNetworkNodeUrl) if err != nil { return nil, err } From 966b874a7df4b4bf38f4a8e30857a20c236b181d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 15:23:26 -0800 Subject: [PATCH 124/127] Update some comments and TODOs --- pkg/appgateserver/cmd/cmd.go | 2 +- pkg/relayer/cmd/cmd.go | 16 +++++++++------- pkg/relayer/proxy/proxy.go | 2 +- pkg/relayer/session/session.go | 5 +++-- proto/pocket/service/relay.proto | 2 ++ 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index a2e71bed6..7a541a511 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -65,7 +65,7 @@ relays to the AppGate server and function as an Application, provided that: // Cosmos flags cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "The URL of the comet tcp endpoint to communicate with the pocket blockchain") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly and uses flagQueryNodeUrl underneath") return cmd } diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index e5a5d1729..ea3df89b6 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -39,18 +39,19 @@ func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", Short: "Run a relay miner", - // TODO_TECHDEBT: add a longer long description. Long: `Run a relay miner. The relay miner process configures and starts relay servers for each service the supplier actor identified by --signing-key is -staked for (configured on-chain). Relay requests received by the relay servers -are validated and proxied to their respective service endpoints. The responses +staked for (configured on-chain). + +Relay requests received by the relay servers are validated and proxied to their +respective service endpoints, maintained by the relayer off-chain. The responses are then signed and sent back to the requesting application. For each successfully served relay, the miner will hash and compare its difficulty against an on-chain threshold. If the difficulty is sufficient, it is applicable to relay volume and therefore rewards. Such relays are inserted into and persisted via an SMT KV store. The miner will monitor the current block height and periodically -submit claim and proof messages according to the protocol as sessions become eligable +submit claim and proof messages according to the protocol as sessions become eligible for such operations.`, RunE: runRelayer, } @@ -65,7 +66,7 @@ for such operations.`, // Communication flags cmd.Flags().StringVar(&flagNetworkNodeUrl, "network-node", omittedDefaultFlagValue, "tcp://: to a pocket node that gossips transactions throughout the network (may or may not be the sequencer") cmd.Flags().StringVar(&flagQueryNodeUrl, "query-node", omittedDefaultFlagValue, "tcp://: to a full pocket node for reading data and listening for on-chain events") - cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly") + cmd.Flags().String(cosmosflags.FlagNode, omittedDefaultFlagValue, "registering the default cosmos node flag; needed to initialize the cosmostx and query contexts correctly and uses flagQueryNodeUrl underneath") return cmd } @@ -351,8 +352,9 @@ func supplyRelayerProxy( return nil, err } - // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented an a local LLM node - // is supported, this needs to be expanded such that a single relayer can proxy to multiple services at once. + // TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented AND a local LLM RPC service + // is supported on LocalNet, this needs to be expanded to include more than one service. The ability to support + // multiple services is already in place but currently (as seen below) is hardcoded. proxiedServiceEndpoints := map[string]url.URL{ "anvil": *proxyServiceURL, } diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index 28f92576c..b6c2a5f68 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -123,7 +123,7 @@ func NewRelayerProxy( rp.sessionQuerier = sessiontypes.NewQueryClient(clientCtx) rp.applicationQuerier = apptypes.NewQueryClient(clientCtx) rp.keyring = rp.clientCtx.Keyring - rp.ringCache = make(map[string][]ringtypes.Point) + rp.ringCache = make(map[string][]ringtypes.Point) // the key is the appAddress rp.ringCacheMutex = &sync.RWMutex{} for _, opt := range opts { diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index ef89e874c..08ea0f962 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -161,10 +161,11 @@ func (rs *relayerSessionsManager) mapBlockToSessionsToClaim( // Iterate over the sessionsTrees map to get the ones that end at a block height // lower than the current block height. for endBlockHeight, sessionsTreesEndingAtBlockHeight := range rs.sessionsTrees { - // TODO: We need this to be == instead of <= because we don't want to keep sending + // TODO_BLOCKER(@red-0ne): We need this to be == instead of <= because we don't want to keep sending // the same session while waiting the next step. This does not address the case // where the block client misses the target block which should be handled by the - // retry mechanism. + // retry mechanism. See the discussion in the following GitHub thread for next + // steps: https://github.com/pokt-network/poktroll/pull/177/files?show-viewed-files=true&file-filters%5B%5D=#r1391957041 if endBlockHeight == block.Height() { // Iterate over the sessionsTrees that end at this block height (or // less) and add them to the list of sessionTrees to be published. diff --git a/proto/pocket/service/relay.proto b/proto/pocket/service/relay.proto index dc269fad0..c9f0d5d77 100644 --- a/proto/pocket/service/relay.proto +++ b/proto/pocket/service/relay.proto @@ -34,6 +34,8 @@ message RelayRequest { } } +// TODO_TECHDEBT(#189, @h5law): See discussion related to #189 on how/why JSONRPC should be refactored altogether. + // JSONRPCRequestPayload contains the payload for a JSON-RPC request. // See https://www.jsonrpc.org/specification#request_object for more details. message JSONRPCRequestPayload { From c726dfc0287aa0e7c2378209a79a3c885b7f242c Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 15:26:57 -0800 Subject: [PATCH 125/127] Added a couple more comments --- pkg/appgateserver/server.go | 2 ++ pkg/relayer/proxy/error_reply.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go index a85d5fae3..fe4f15000 100644 --- a/pkg/appgateserver/server.go +++ b/pkg/appgateserver/server.go @@ -225,6 +225,8 @@ func (app *appGateServer) replyWithError(writer http.ResponseWriter, err error) relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ + // TODO_BLOCKER(@red-0ne): This MUST match the Id provided by the request. + // If JSON-RPC request is not unmarshaled yet (i.e. can't extract ID), it SHOULD be a random ID. Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ diff --git a/pkg/relayer/proxy/error_reply.go b/pkg/relayer/proxy/error_reply.go index c6c5606e6..617f3b917 100644 --- a/pkg/relayer/proxy/error_reply.go +++ b/pkg/relayer/proxy/error_reply.go @@ -16,6 +16,8 @@ func (jsrv *jsonRPCServer) replyWithError(writer http.ResponseWriter, err error) relayResponse := &types.RelayResponse{ Payload: &types.RelayResponse_JsonRpcPayload{ JsonRpcPayload: &types.JSONRPCResponsePayload{ + // TODO_BLOCKER(@red-0ne): This MUST match the Id provided by the request. + // If JSON-RPC request is not unmarshaled yet (i.e. can't extract ID), it SHOULD be a random ID. Id: 0, Jsonrpc: "2.0", Error: &types.JSONRPCResponseError{ From c36db928217a21b1255d0918d8f7590e94efa85a Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 15 Nov 2023 15:41:27 -0800 Subject: [PATCH 126/127] More tiny comment updates --- pkg/relayer/cmd/cmd.go | 5 ++--- pkg/relayer/proxy/server_builder.go | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index ea3df89b6..4f43c4ff2 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -344,9 +344,8 @@ func supplyRelayerProxy( deps depinject.Config, _ *cobra.Command, ) (depinject.Config, error) { - // TODO_BLOCKER:(#137): This MUST be populated via the `relayer.json` config file - // TODO_UPNEXT(@okdas): this hostname should be updated to match that of the - // in-tilt anvil service. + // TODO_BLOCKER:(#137, @red-0ne): This MUST be populated via the `relayer.json` config file + // TODO_UPNEXT(@okdas): this hostname should be updated to match that of the in-tilt anvil service. proxyServiceURL, err := url.Parse("http://localhost:8547/") if err != nil { return nil, err diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index 4238c1ce3..3846ac3af 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -20,7 +20,6 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { return err } - // TODO_DISCUSS: is there a reason not to assign rp.supplierAddress here? supplierAddress, err := supplierKey.GetAddress() if err != nil { return err From 27ddcff555756c9f01c3a4475c4a2820ce29ca48 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 16 Nov 2023 11:02:30 +0100 Subject: [PATCH 127/127] chore: review feedback improvements Co-authored-by: Daniel Olshansky --- e2e/tests/init_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 88a8afff4..c7492903b 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -24,7 +24,7 @@ import ( suppliertypes "github.com/pokt-network/poktroll/x/supplier/types" ) -// TODO_TECHDEBT(#179): Once relayminer and appgateserver are running in tilt, +// TODO_TECHDEBT(@okdas): Once relayminer and appgateserver are running in tilt, // use their respective in-tilt hostnames and run E2E tests in tilt. This // should match the on-chain advertised endpoint for the service with the // given serviceId.