[Utility] trustless relays servicer token validation

Makefile

@@ -411,6 +411,10 @@ test_persistence: ## Run all go unit tests in the Persistence module
 test_persistence_state_hash: ## Run all go unit tests in the Persistence module related to the state hash
 	go test ${VERBOSE_TEST} -count=1 -tags=test -run TestStateHash ./persistence/...
 
+.PHONY: test_servicer_relay
+test_servicer_relay: ## Run all go unit tests related to servicer relays
+	go test ${VERBOSE_TEST} -count=1 -tags=test ./utility/servicer -run TestRelay
+
 .PHONY: test_p2p
 test_p2p: ## Run all p2p related tests
 	go test ${VERBOSE_TEST} -count=1 -tags=test ./p2p/...

persistence/docs/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.0.58] - 2023-06-12
+
+- Add place-holder for local context and servicer token usage support methods
+
 ## [0.0.0.57] - 2023-06-06
 
 - Uses ":memory:" to signify when connecting to an in-memory database

persistence/local/module.go

@@ -0,0 +1,80 @@
+package local
+
+import (
+	"math/big"
+
+	"github.com/pokt-network/pocket/logger"
+	coreTypes "github.com/pokt-network/pocket/shared/core/types"
+	"github.com/pokt-network/pocket/shared/modules"
+	"github.com/pokt-network/pocket/shared/modules/base_modules"
+)
+
+const (
+	LocalModuleName = "local"
+)
+
+var _ modules.PersistenceLocalContext = &persistenceLocalContext{}
+
+type persistenceLocalContext struct {
+	base_modules.IntegratableModule
+
+	logger       *modules.Logger
+	databasePath string
+}
+
+func WithLocalContextConfig(databasePath string) modules.ModuleOption {
+	return func(m modules.InitializableModule) {
+		if plc, ok := m.(*persistenceLocalContext); ok {
+			plc.databasePath = databasePath
+		}
+	}
+}
+
+func CreateLocalContext(bus modules.Bus, options ...modules.ModuleOption) (modules.PersistenceLocalContext, error) {
+	m, err := new(persistenceLocalContext).Create(bus, options...)
+	if err != nil {
+		return nil, err
+	}
+	return m.(modules.PersistenceLocalContext), nil
+}
+
+func (*persistenceLocalContext) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) {
+	m := &persistenceLocalContext{}
+
+	for _, option := range options {
+		option(m)
+	}
+
+	bus.RegisterModule(m)
+
+	m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName())
+
+	return m, nil
+}
+
+func (m *persistenceLocalContext) GetModuleName() string {
+	return LocalModuleName
+}
+
+// INCOMPLETE(#826): implement this
+func (m *persistenceLocalContext) Start() error {
+	return nil
+}
+
+// INCOMPLETE(#826): implement this
+func (m *persistenceLocalContext) Stop() error {
+	return nil
+}
+
+// INCOMPLETE(#826): implement this
+// OPTIMIZE: both the relay and the response can be large structures: we may need to truncate the stored values
+// StoreServicedRelay implements the PersistenceLocalContext interface
+func (local *persistenceLocalContext) StoreServicedRelay(session *coreTypes.Session, relayDigest, relayReqResBytes []byte) error {
+	return nil
+}
+
+// INCOMPLETE(#826): implement this
+// GetSessionTokensUsed implements the PersistenceLocalContext interface
+func (local *persistenceLocalContext) GetSessionTokensUsed(*coreTypes.Session) (*big.Int, error) {
+	return nil, nil
+}

persistence/module.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pokt-network/pocket/logger"
 	"github.com/pokt-network/pocket/persistence/blockstore"
 	"github.com/pokt-network/pocket/persistence/indexer"
+	"github.com/pokt-network/pocket/persistence/local"
 	"github.com/pokt-network/pocket/runtime/configs"
 	"github.com/pokt-network/pocket/runtime/genesis"
 	"github.com/pokt-network/pocket/shared/modules"
@@ -242,6 +243,12 @@ func (m *persistenceModule) NewWriteContext() modules.PersistenceRWContext {
 	return m.writeContext
 }
 
+// INCOMPLETE(#826): implement this
+// GetLocalContext returns a new local context for storing off-chain, i.e. node-specific, data.
+func (m *persistenceModule) GetLocalContext() (modules.PersistenceLocalContext, error) {
+	return local.CreateLocalContext(m.GetBus())
+}
+
 // HACK(olshansky): Simplify and externalize the logic for whether genesis should be populated and
 //
 //	move the if logic out of this file.

persistence/servicer.go

@@ -2,6 +2,7 @@ package persistence
 
 import (
 	"encoding/hex"
+	"math/big"
 
 	"github.com/pokt-network/pocket/persistence/types"
 	coreTypes "github.com/pokt-network/pocket/shared/core/types"
@@ -79,3 +80,15 @@ func (p *PostgresContext) SetServicerPauseHeight(address []byte, height int64) e
 func (p *PostgresContext) GetServicerOutputAddress(operator []byte, height int64) (output []byte, err error) {
 	return p.GetActorOutputAddress(types.ServicerActor, operator, height)
 }
+
+// INCOMPLETE: implement this
+// DISCUSS: both the relay and the response can be large structures: we may need to truncate the stored values
+func (p *PostgresContext) RecordRelayService(applicationAddress string, key []byte, relay *coreTypes.Relay, response *coreTypes.RelayResponse) error {
+	return nil
+}
+
+// INCOMPLETE: implement this
+// GetServicerTokenUsage returns the number of tokens used by the servicer in the current session, i.e. for the application associated with the session
+func (p *PostgresContext) GetServicerTokenUsage(session *coreTypes.Session) (*big.Int, error) {
+	return nil, nil
+}

rpc/doc/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.0.24] - 2023-06-21
+
+- Update handlers to use the new relay payload types
+
 ## [0.0.0.23] - 2023-06-19
 
 - Remove AAT reference from handlers

rpc/v1/openapi.yaml

@@ -1696,11 +1696,12 @@ components:
     Payload:
       type: object
       required:
-        - method
         - jsonrpc
+        - method
       properties:
         id:
           type: string
+          format: byte
         jsonrpc:
           type: string
         method:

runtime/configs/config.go

@@ -36,7 +36,10 @@ type Config struct {
 
 // ParseConfig parses the config file and returns a Config struct
 func ParseConfig(cfgFile string) *Config {
-	config := NewDefaultConfig()
+	defaultCfg := NewDefaultConfig()
+
+	// Bind environment variables so POCKET_ env vars work without having to set in config file
+	bindViperToEnvVariables(defaultCfg)
 
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
@@ -48,13 +51,6 @@ func ParseConfig(cfgFile string) *Config {
 		viper.SetConfigType("json")          // REQUIRED if the config file does not have the extension in the name
 	}
 
-	// The lines below allow for environment variables configuration (12 factor app)
-	// Eg: POCKET_CONSENSUS_PRIVATE_KEY=somekey would override `consensus.private_key` in config.json
-	// If the key is not set in the config, the env var will not be used.
-	viper.SetEnvPrefix("POCKET")
-	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-	viper.AutomaticEnv()
-
 	verbose := viper.GetBool("verbose")
 
 	if err := viper.ReadInConfig(); err != nil {
@@ -73,11 +69,30 @@ func ParseConfig(cfgFile string) *Config {
 		}
 	}
 
+	customConfig := getConfigFromViper()
+	adjustConfigDefaults(customConfig, defaultCfg)
+	setViperDefaultConfig(defaultCfg)
+
+	// Read the configuration again, this time with adjusted default values set on viper. The first read was needed
+	//	to be able to make adjustments to the defaults based on the user-defined configuration.
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatalf("[ERROR] failed to read adjusted default config %s", err.Error())
+	}
+
+	// Return the config resulting from merging of custom config and adjusted default config
+	return getConfigFromViper()
+}
+
+// getConfigFromViper returns the configuration from viper. It is used instead of simply using viper unmarshalling
+// to allow a fallback to json.Unmarshal if necessary.
+func getConfigFromViper() *Config {
 	decoderConfig := func(dc *mapstructure.DecoderConfig) {
 		// This is to leverage the `json` struct tags without having to add `mapstructure` ones.
 		// Until we have complex use cases, this should work just fine.
 		dc.TagName = "json"
 	}
+
+	var config *Config
 	// Detect if we need to use json.Unmarshal instead of viper.Unmarshal
 	if err := viper.Unmarshal(&config, decoderConfig); err != nil {
 		cfgData := viper.AllSettings()
@@ -92,15 +107,60 @@ func ParseConfig(cfgFile string) *Config {
 	return config
 }
 
-// setViperDefaults this is a hacky way to set the default values for Viper so env var overrides work.
+// adjustConfigDefaults performs the necessary adjustments on the default configuration depending on the provided custom configuration.
+//
+//	This adjust of default configuration is needed to address cases where simply overriding the defaults
+//	with the user-supplied values is not sufficient. An example of this is when the custom configuration
+//	enables Fisherman, which means a Servicer should not be enabled on config, and therefore the default
+//	configuration should not include a default Servicer.
+func adjustConfigDefaults(customCfg, defaultCfg *Config) {
+	if customCfg == nil {
+		return
+	}
+
+	// Disable default servicer if the custom configuration has fisherman enabled.
+	if customCfg.Fisherman != nil && customCfg.Fisherman.Enabled {
+		defaultCfg.Servicer = &ServicerConfig{}
+	}
+}
+
+// bindViperToEnvVariables binds viper to environment variables with names dervied from the keys in the provided config.
+//
+//	This is needed so environment variable overrides work
+//
 // DISCUSS: is there a better way to do this?
-func setViperDefaults(cfg *Config) {
-	// convert the config struct to a map with the json tags as keys
+func bindViperToEnvVariables(cfg *Config) {
+	// The lines below allow for environment variables configuration (12 factor app)
+	// Eg: POCKET_CONSENSUS_PRIVATE_KEY=somekey would override `consensus.private_key` in config.json
+	// If the key is not set in the config, the env var will not be used.
+	viper.SetEnvPrefix("POCKET")
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AutomaticEnv()
+
 	cfgData, err := json.Marshal(cfg)
+	if err != nil {
+		log.Fatalf("[ERROR] failed to marshal config %s", err.Error())
+	}
 
+	var cfgMap map[string]any
+	if err := json.Unmarshal(cfgData, &cfgMap); err != nil {
+		log.Fatalf("[ERROR] failed to unmarshal config %s", err.Error())
+	}
+
+	// ADDTEST: test scenarios related to environment variables, e.g. override of default/configured values
+	for envVar := range cfgMap {
+		if err := viper.BindEnv(envVar); err != nil {
+			log.Fatalf("[ERROR] failed to bind env. var. %s: %s", envVar, err.Error())
+		}
+	}
+}
+
+func setViperDefaultConfig(cfg *Config) {
+	cfgData, err := json.Marshal(cfg)
 	if err != nil {
 		log.Fatalf("[ERROR] failed to marshal config %s", err.Error())
 	}
+
 	var cfgMap map[string]any
 	if err := json.Unmarshal(cfgData, &cfgMap); err != nil {
 		log.Fatalf("[ERROR] failed to unmarshal config %s", err.Error())
@@ -157,17 +217,14 @@ func NewDefaultConfig(options ...func(*Config)) *Config {
 			VaultMountPath: defaults.DefaultKeybaseVaultMountPath,
 		},
 		Validator: &ValidatorConfig{},
-		Servicer:  &ServicerConfig{},
+		Servicer:  defaultServicerConfig(),
 		Fisherman: &FishermanConfig{},
 	}
 
 	for _, option := range options {
 		option(cfg)
 	}
 
-	// set Viper defaults so POCKET_ env vars work without having to set in config file
-	setViperDefaults(cfg)
-
 	return cfg
 }
 
@@ -211,3 +268,21 @@ func CreateTempConfig(cfg *Config) (*Config, error) {
 
 	return ParseConfig(tmpfile.Name()), nil
 }
+
+func defaultServicerConfig() *ServicerConfig {
+	return &ServicerConfig{
+		Enabled:                   true,
+		RelayMiningVolumeAccuracy: 0.2,
+		Services: map[string]*ServiceConfig{
+			// TODO(#831): Design how Chain/Service IDs should be described/defined.
+			"POKT-LocalNet": {
+				Url:         "http://localhost",
+				TimeoutMsec: 5000,
+				BasicAuth: &BasicAuth{
+					UserName: "user",
+					Password: "password",
+				},
+			},
+		},
+	}
+}

runtime/configs/proto/persistence_config.proto

@@ -16,4 +16,6 @@ message PersistenceConfig {
   string max_conn_lifetime = 8; // See pkg.go.dev/time#ParseDuration for reference
   string max_conn_idle_time = 9; // See pkg.go.dev/time#ParseDuration for reference
   string health_check_period = 10; // See pkg.go.dev/time#ParseDuration for reference
+  // TODO: `local_database_path` may need to be expanded to multiple stores depending on how usage evolves
+  string local_database_path = 11; // The path used to store local, i.e. off-chain and node-specific, data.
 }

runtime/configs/proto/servicer_config.proto

@@ -5,11 +5,34 @@ package configs;
 
 option go_package = "github.com/pokt-network/pocket/runtime/configs";
 
+// TODO: Reevalute whether each utility actor should contain address/pubKey configs or if it should be shared
 // ServicerConfig defines the configuration for the node acting as a servicer. Servicers earn rewards for providing Web3 access over a function of volume and quality
 message ServicerConfig {
   // Enabled defines whether or not the node is a servicer.
   bool enabled = 1;
   string public_key = 2;
   string address = 3;
-  repeated string chains = 4;
+  map<string, ServiceConfig> services = 4;
+
+  // relay_mining_volume_accuracy is a parameter used to adjust the calculated number of service tokens for an application.
+  // It is introduced to minimize the chance of under-utilization of application's tokens, while removing the overhead of
+  // communication between servicers which would be necessary otherwise.
+  // See the following for more details:
+  //	https://arxiv.org/abs/2305.10672
+  double relay_mining_volume_accuracy = 5;
+}
+
+// ServiceConfig holds configurations related to where/how the application/client can access the backing RPC service. It is analogous to "ChainConfig" in v0 but can support any RPC service.
+message ServiceConfig {
+  string url = 1; // url specifies the URL at which the service is provided/requested
+  uint64 timeout_msec = 2; // timeout specifes the maximum amount of time, in milliseconds, to allow for the service to return a response
+  BasicAuth basic_auth = 3; // optional: basic authentication for HTTP services.
+}
+
+// BasicAuth stores authentication data for HTTP services
+// When supplied, this data will be used as specified by the HTTP basic authentication scheme.
+message BasicAuth {
+  string user_name = 1;
+  // IMPROVE: enforce encryption of plaintext password
+  string password = 2;
 }

runtime/docs/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.0.43] - 2023-06-17
+
+- Add a new ServiceConfig field to servicer config
+
 ## [0.0.0.42] - 2023-06-13
 
 - Append "Hostname" to validator endpoint hostname constants