[Off-Chain, Scalability] Query client caching & history #985

bryanchriswhite · 2024-12-05T12:41:18Z

Summary

Adds caching to the query clients.

---
title: Client caching options
---

classDiagram-v2

class ParamsQuerier__P__sdk_Msg {
    <<interface>>
    GetParams(ctx context.Context) (params P, err error)
    GetParamsAtHeight(ctx context.Context, height int64) (params P, err error)
}

class paramsQuerier__P__sdk_Msg {
    <<interface>>
    GetParams(ctx context.Context) (P, error)
}

class cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier:::cacheImpl {
    queryClient Q__paramsQuerier[P]
    paramsCache HistoricalQueryCache[P]
    ...
}
ParamsQuerier__P__sdk_Msg --|> cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier
cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier --* InMemoryCache: EITHER
cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier ..* BadgerCache: OR
cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier --> paramsQuerier__P__sdk_Msg


class QueryCache__T__any {
    <<interface>>
    Get(key string) (value T, err error)
    Set(key string, value T) (err error)
    Delete(key string)
    Clear()
}

class HistoricalQueryCache__T__any {
    <<interface>>
    QueryCache__T
    GetAtHeight(key string, heigt int64) (value T, err error)
    SetAtHeight(key string, value T, heigt int64) (err error)
}
HistoricalQueryCache__T__any --|> QueryCache__T__any

class InMemoryCache:::cacheImpl
InMemoryCache --|> HistoricalQueryCache__T__any

class BadgerCache:::cacheImpl
BadgerCache --|> HistoricalQueryCache__T__any

%% TODO_IN_THIS_COMMIT: uncomment classDef in docs - github can't parse it.
%%classDef cacheImpl fill:#074,color:#fff;

---
title: Example client integration (session module)
---

classDiagram-v2

class ParamsQuerier__P__sdk_Msg {
    <<interface>>
    GetParams(ctx context.Context) (params P, err error)
    GetParamsAtHeight(ctx context.Context, height int64) (params P, err error)
}


class cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier {
    queryClient Q__paramsQuerier[P]
    paramsCache HistoricalQueryCache[P]
    ...
}
ParamsQuerier__P__sdk_Msg --|> cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier

class QueryCache__T__any {
    <<interface>>
    Get(key string) (value T, err error)
    Set(key string, value T) (err error)
    Delete(key string)
    Clear()
}

class HistoricalQueryCache__T__any {
    <<interface>>
    QueryCache__T__any
    GetAtHeight(key string, heigt int64) (value T, err error)
    SetAtHeight(key string, value T, heigt int64) (err error)
}
HistoricalQueryCache__T__any --|> QueryCache__T__any

cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier --* HistoricalQueryCache__T__any

class SessionQuerier {
    <<interface>>
    ParamsQuerier__P__sdk_Msg
}
SessionQuerier --|> ParamsQuerier__P__sdk_Msg
SessionQuerier --|> sessionQuerier

class sessionQuerier {
    ParamsQuerier__P__sdk_Msg
    sessionCache QueryCache__T__any
    ...
}
sessionQuerier --* cachedParamsQuerier__P__sdk_Msg__Q__paramsQuerier
sessionQuerier --* QueryCache__T__any: (sessionCache)

class paramsQuerier__P__sdk_Msg {
    <<interface>>
    GetParams(ctx context.Context) (P, error)
}
HistoricalQueryCache__T__any ..> paramsQuerier__P__sdk_Msg
sessionQuerier ..|> paramsQuerier__P__sdk_Msg: (non-caching interface adapter)

Issue

[Off-Chain] ModuleParamsClient & Historical Params #543

Type of change

Select one or more from the following:

New feature, functionality or library
Consensus breaking; add the consensus-breaking label if so. See [Infra] Automatically add the consensus-breaking label #791 for details
Bug fix
Code health or cleanup
Documentation
Other (specify)

Testing

Documentation: make docusaurus_start; only needed if you make doc changes
Unit Tests: make go_develop_and_test
LocalNet E2E Tests: make test_e2e
DevNet E2E Tests: Add the devnet-test-e2e label to the PR.

Sanity Checklist

I have tested my changes using the available tooling
I have commented my code
I have performed a self-review of my own code; both comments & source code
I create and reference any new tickets, if applicable
I have left TODOs throughout the codebase, if applicable

github-actions · 2024-12-05T12:41:39Z

pkg/client/query/sharedquerier.go

 	); err != nil {
 		return nil, err
 	}

-	querier.sharedQuerier = sharedtypes.NewQueryClient(querier.clientConn)
+	sq.paramsCache = cache.NewInMemoryCache[*sharedtypes.Params](
+		// TODO_IN_THIS_COMMIT: extract to constants.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract to constants.

github-actions · 2024-12-05T12:41:39Z

pkg/client/query/sharedquerier.go

+	sq.paramsCache = cache.NewInMemoryCache[*sharedtypes.Params](
+		// TODO_IN_THIS_COMMIT: extract to constants.
+		cache.WithHistoricalMode(100),
+		// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/sharedquerier.go

+		// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...
+		//cache.WithMaxSize(1),
+		cache.WithEvictionPolicy(cache.FirstInFirstOut),
+		// TODO_IN_THIS_COMMIT: extract to a constant.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract to a constant.

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/cache/config.go

+// EvictionPolicy determines how items are removed when cache is full
+type EvictionPolicy string
+
+// TODO_IN_THIS_COMMIT: refactor to an enum.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: refactor to an enum.

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/cache/config.go

+	// historical is whether the cache will cache a single value for each key
+	// (false) or whether it will cache a history of values for each key (true).
+	historical bool
+	// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/sessionquerier.go

+
+	// Initialize params cache with minimal configuration since we only need latest
+	sq.paramsCache = cache.NewInMemoryCache[*sessiontypes.Params](
+		// TODO_IN_THIS_COMMIT: extract to a constant.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract to a constant.

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/sessionquerier.go

+	sq.paramsCache = cache.NewInMemoryCache[*sessiontypes.Params](
+		// TODO_IN_THIS_COMMIT: extract to a constant.
+		cache.WithHistoricalMode(100),
+		// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/sessionquerier.go

+		// TODO_IN_THIS_COMMIT: reconcile the fact that MaxSize doesn't apply to historical mode...
+		//cache.WithMaxSize(1),
+		cache.WithEvictionPolicy(cache.FirstInFirstOut),
+		// TODO_IN_THIS_COMMIT: extract to a constant.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract to a constant.

github-actions · 2024-12-05T12:41:40Z

pkg/client/query/sessionquerier.go

 	); err != nil {
 		return nil, err
 	}

-	sessq.sessionQuerier = sessiontypes.NewQueryClient(sessq.clientConn)
+	// TODO_IN_THIS_COMMIT: kick off a goroutine that subscribes to params updates and populates the cache.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: kick off a goroutine that subscribes to params updates and populates the cache.

github-actions · 2024-12-05T12:41:41Z

pkg/client/block/client.go

@@ -187,6 +187,12 @@ func (b *blockReplayClient) queryLatestBlock(
 	errCh := make(chan error)

 	go func() {
+		// TODO_IN_THIS_COMMIT: extract labels (and values?) to constants.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract labels (and values?) to constants.

github-actions

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

linter-name (fail-on-found)

github-actions · 2024-12-06T17:21:59Z

pkg/client/query/proofquerier.go

 	prooftypes "github.com/pokt-network/poktroll/x/proof/types"
 )

+// TODO_IN_THIS_COMMIT: comment explaining why we can't use client.ProofQueryClient;


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: comment explaining why we can't use client.ProofQueryClient;

github-actions · 2024-12-06T17:21:59Z

pkg/client/query/proofquerier.go

+func NewProofQuerier(
+	deps depinject.Config,
+	paramsQuerierOpts ...ParamsQuerierOptionFn,
+	// TODO_IN_THIS_COMMIT: comment explaining why we can't use client.ProofQueryClient;


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: comment explaining why we can't use client.ProofQueryClient;

github-actions · 2024-12-06T17:21:59Z

pkg/client/query/proofquerier.go

+	}
+
+	querier := &proofQuerier{
+		// TODO_IN_THIS_COMMIT: extract this to an option.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract this to an option.

github-actions · 2024-12-06T17:21:59Z

pkg/client/query/sharedquerier.go

+	}
+
+	sq := &sharedQuerier{
+		// TODO_IN_THIS_COMMIT: extract this to an option.


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: extract this to an option.

github-actions · 2024-12-06T17:21:59Z

pkg/client/query/paramsquerier.go

+	"github.com/pokt-network/poktroll/pkg/polylog"
+)
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-06T17:22:02Z

x/proof/types/query_client.go

+	return NewQueryClient(conn).(ProofQueryClient)
+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: investigate generalization...

github-actions · 2024-12-06T17:22:02Z

x/proof/types/query_client.go

+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-06T17:22:02Z

x/shared/types/query_client.go

+	gogogrpc "github.com/cosmos/gogoproto/grpc"
+)
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-06T17:22:02Z

x/shared/types/query_client.go

+	GetParams(context.Context) (*Params, error)
+}
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-06T17:22:02Z

x/shared/types/query_client.go

+	return NewQueryClient(conn).(SharedQueryClient)
+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: investigate generalization...

github-actions · 2024-12-09T16:41:14Z

x/supplier/types/query_client.go

+	gogogrpc "github.com/cosmos/gogoproto/grpc"
+)
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:14Z

x/supplier/types/query_client.go

+	GetParams(context.Context) (*Params, error)
+}
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:14Z

x/supplier/types/query_client.go

+	return NewQueryClient(conn).(SupplierQueryClient)
+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: investigate generalization...

github-actions · 2024-12-09T16:41:14Z

x/supplier/types/query_client.go

+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:14Z

x/shared/types/query_client.go

+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:15Z

x/service/types/query_client.go

+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:15Z

x/session/types/query_client.go

+	gogogrpc "github.com/cosmos/gogoproto/grpc"
+)
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:15Z

x/session/types/query_client.go

+	GetParams(context.Context) (*Params, error)
+}
+
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

github-actions · 2024-12-09T16:41:15Z

x/session/types/query_client.go

+	return NewQueryClient(conn).(SessionQueryClient)
+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: investigate generalization...

github-actions · 2024-12-09T16:41:15Z

x/session/types/query_client.go

+}
+
+// TODO_IN_THIS_COMMIT: investigate generalization...
+// TODO_IN_THIS_COMMIT: godoc...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: godoc...

bryanchriswhite · 2024-12-09T17:00:24Z

go.mod

@@ -12,6 +12,8 @@ go 1.23.0
 // github.com/pokt-network/smt/kvstore/pebble => ../smt/kvstore/pebble
 // )

+replace nhooyr.io/websocket => github.com/coder/websocket v1.8.6


bryanchriswhite · 2024-12-09T17:00:37Z

go.sum

bryanchriswhite · 2024-12-09T17:00:51Z

localnet/kubernetes/values-relayminer-common.yaml

github-actions · 2024-12-10T13:12:53Z

pkg/client/query/options.go

+		CacheOpts: []cache.CacheOption{
+			// TODO_IN_THIS_COMMIT: extract to constants.
+			cache.WithHistoricalMode(100),
+			// TODO_IN_THIS_COMMIT: reconcile the fact that MaxItems doesn't apply to historical mode...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: reconcile the fact that MaxItems doesn't apply to historical mode...

github-actions · 2024-12-10T13:53:05Z

pkg/client/query/servicequerier.go

@@ -51,6 +72,8 @@ func (servq *serviceQuerier) GetService(
 		Id: serviceId,
 	}

+	// TODO_IN_THIS_COMMIT: historically cache services...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: historically cache services...

github-actions · 2024-12-10T13:53:05Z

pkg/client/query/servicequerier.go

@@ -71,6 +94,8 @@ func (servq *serviceQuerier) GetServiceRelayDifficulty(
 		ServiceId: serviceId,
 	}

+	// TODO_IN_THIS_COMMIT: historically cache relay mining difficulties...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: historically cache relay mining difficulties...

github-actions · 2024-12-10T13:53:05Z

pkg/client/query/options.go

+		CacheOpts: []cache.CacheOption{
+			// TODO_IN_THIS_COMMIT: extract to constants.
+			cache.WithHistoricalMode(100),
+			// TODO_IN_THIS_COMMIT: reconcile the fact that MaxKeys doesn't apply to historical mode...


[linter-name (fail-on-found)] _{reported by reviewdog 🐶}
// TODO_IN_THIS_COMMIT: reconcile the fact that MaxKeys doesn't apply to historical mode...

bryanchriswhite added off-chain Off-chain business logic scalability p0 Top priority labels Dec 5, 2024

bryanchriswhite added this to the Shannon Beta TestNet Support milestone Dec 5, 2024

bryanchriswhite self-assigned this Dec 5, 2024

github-actions bot reviewed Dec 5, 2024

View reviewed changes

bryanchriswhite added the relayminer Changes related to the Relayminer label Dec 5, 2024

github-actions bot reviewed Dec 6, 2024

View reviewed changes

github-actions bot reviewed Dec 9, 2024

View reviewed changes

bryanchriswhite commented Dec 9, 2024

View reviewed changes

go.sum Outdated

Copy link

Contributor Author

bryanchriswhite Dec 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

revert

bryanchriswhite commented Dec 9, 2024

View reviewed changes

localnet/kubernetes/values-relayminer-common.yaml Outdated

Copy link

Contributor Author

bryanchriswhite Dec 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

revert

github-actions bot reviewed Dec 10, 2024

View reviewed changes

bryanchriswhite added 16 commits December 10, 2024 15:54

chore: add QueryCache interface

0ba527a

chore: add HistoricalQueryCache interface

af418e7

feat: add InMemoryCache implementation

6df478a

chore: add ParamsQuerier interface

0e5b214

feat: add ParamsQuerier

4f989a1

refactor: app query client

b4beeed

fix: application transfer integration test

12e3dd3

refactor: proof query client

d7e6a10

refactor: service query client

f25bcc5

refactor: session query client

c8af216

refactor: shared query client

3d98b18

refactor: supplier query client

9388489

fixup: UNKNOWN - tokenomics expected keepers GetParams()

aadbd8f

wip: badger cache

cdf11f8

chore: client query metrics

16acda7

revert: up next

b07a8b8

bryanchriswhite force-pushed the issues/543/client-caching branch from 6f5f1c3 to b07a8b8 Compare December 10, 2024 14:55

bryanchriswhite closed this Dec 17, 2024

[Off-Chain, Scalability] Query client caching & history #985

[Off-Chain, Scalability] Query client caching & history #985

Conversation

bryanchriswhite commented Dec 5, 2024 • edited Loading

Summary

Issue

Type of change

Testing

Sanity Checklist

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot Dec 5, 2024

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 6, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 9, 2024

Choose a reason for hiding this comment

bryanchriswhite Dec 9, 2024

Choose a reason for hiding this comment

bryanchriswhite Dec 9, 2024

Choose a reason for hiding this comment

bryanchriswhite Dec 9, 2024

Choose a reason for hiding this comment

github-actions bot Dec 10, 2024

Choose a reason for hiding this comment

github-actions bot Dec 10, 2024

bryanchriswhite commented Dec 5, 2024 •

edited

Loading