diff --git a/.gitignore b/.gitignore index a75541c..d7095d7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ .idea -config.yml +config.json config_* prometheus \ No newline at end of file diff --git a/README.md b/README.md index c7a6fa4..45bb528 100644 --- a/README.md +++ b/README.md @@ -39,47 +39,69 @@ go test -v ./... For local development and testing, you can run the application with: ```console -DEBUG=true go run . --config config.yml +DEBUG=true go run . --config config.json ``` -## Configuration +Additionally, to load configuration from an environment variable, use the `--env` flag. Ensure the `GATEWAY_CONFIG` environment variable is set with the main configuration data. -A main YAML configuration (`config.yml`) specifies the metrics server port and multiple gateways, each with its own `.yml` configuration file: +```console +DEBUG=true go run . --env +``` -```yaml -metrics: - port: 9090 # Port for Prometheus metrics, served on /metrics and / +## Configuration -gateways: - - config-file: "config_holesky.yml" - name: "Holesky gateway" - - config-file: "config_sepolia.yml" - name: "Sepolia gateway" +The main configuration has been updated to use JSON format (`config.json`). It specifies the metrics server port and multiple gateways, each with its own JSON configuration file: + +```json +{ + "metrics": { + "port": 9090 + }, + "port": 4000, + "gateways": [ + { + "configFile": "config_holesky.json", + "name": "Holesky gateway" + }, + { + "configFile": "config_sepolia.json", + "name": "Sepolia gateway" + } + ] +} ``` -Each `.yml` configuration file for the gateways can specify detailed settings for proxy behavior, health checks, and target node providers. Here is an example of what these individual gateway configuration files can contain: - -```yaml -proxy: - port: "3000" # Port for RPC gateway - upstreamTimeout: "1s" # When is a request considered timed out - -healthChecks: - interval: "5s" # How often to perform health checks - timeout: "1s" # Timeout duration for health checks - failureThreshold: 2 # Failed checks until a target is marked unhealthy - successThreshold: 1 # Successes required to mark a target healthy again - -targets: # Failover order is determined by the list order - - name: "Cloudflare" - connection: - http: - url: "https://cloudflare-eth.com" - - name: "Alchemy" - connection: - http: - url: "https://alchemy.com/rpc/" +Each JSON configuration file for the gateways can specify detailed settings for proxy behavior, health checks, and target node providers. Here is an example of what these individual gateway configuration files might contain: + +```json +{ + "proxy": { + "port": "3000", + "upstreamTimeout": "1s" + }, + "healthChecks": { + "interval": "5s", + "timeout": "1s", + "failureThreshold": 2, + "successThreshold": 1 + }, + "targets": [ + { + "name": "Cloudflare", + "connection": { + "http": { + "url": "https://cloudflare-eth.com" + } + } + }, + { + "name": "Alchemy", + "connection": { + "http": { + "url": "https://alchemy.com/rpc/" + } + } + } + ] +} ``` - -Any of these configuration files can be also loaded from URL if one is provided as path. ---- diff --git a/example_config.json b/example_config.json new file mode 100644 index 0000000..d4bb5f9 --- /dev/null +++ b/example_config.json @@ -0,0 +1,15 @@ +{ + "metrics": { + "port": 9090 + }, + "gateways": [ + { + "configFile": "config_holesky.json", + "name": "Holesky gateway" + }, + { + "configFile": "config_sepolia.json", + "name": "Sepolia gateway" + } + ] +} diff --git a/example_config.yml b/example_config.yml deleted file mode 100644 index 9804e3e..0000000 --- a/example_config.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -metrics: - port: 9090 # port for prometheus metrics, served on /metrics and / - -gateways: - - configFile: "config_holesky.yml" - name: "Holesky gateway" - - configFile: "config_sepolia.yml" - name: "Sepolia gateway" \ No newline at end of file diff --git a/example_network_config.json b/example_network_config.json new file mode 100644 index 0000000..1f91a7d --- /dev/null +++ b/example_network_config.json @@ -0,0 +1,30 @@ +{ + "proxy": { + "port": "3000", + "upstreamTimeout": "1s" + }, + "healthChecks": { + "interval": "5s", + "timeout": "1s", + "failureThreshold": 2, + "successThreshold": 1 + }, + "targets": [ + { + "name": "Cloudflare", + "connection": { + "http": { + "url": "https://cloudflare-eth.com" + } + } + }, + { + "name": "Alchemy", + "connection": { + "http": { + "url": "https://alchemy.com/rpc/" + } + } + } + ] +} diff --git a/example_network_config.yml b/example_network_config.yml deleted file mode 100644 index 71334ab..0000000 --- a/example_network_config.yml +++ /dev/null @@ -1,20 +0,0 @@ - -proxy: - port: "3000" # port for RPC gateway - upstreamTimeout: "1s" # when is a request considered timed out - -healthChecks: - interval: "5s" # how often to do healthchecks - timeout: "1s" # when should the timeout occur and considered unhealthy - failureThreshold: 2 # how many failed checks until marked as unhealthy - successThreshold: 1 # how many successes to be marked as healthy again - -targets: # the order here determines the failover order - - name: "Cloudflare" - connection: - http: # ws is supported by default, it will be a sticky connection. - url: "https://cloudflare-eth.com" - - name: "Alchemy" - connection: - http: # ws is supported by default, it will be a sticky connection. - url: "https://alchemy.com/rpc/" \ No newline at end of file diff --git a/internal/metrics/config.go b/internal/metrics/config.go index 2620a75..28b4e38 100644 --- a/internal/metrics/config.go +++ b/internal/metrics/config.go @@ -1,5 +1,5 @@ package metrics type Config struct { - Port uint `yaml:"port"` + Port uint `json:"port"` } diff --git a/internal/proxy/config.go b/internal/proxy/config.go index b376100..8016f01 100644 --- a/internal/proxy/config.go +++ b/internal/proxy/config.go @@ -1,19 +1,19 @@ package proxy import ( - "time" + "github.com/sygmaprotocol/rpc-gateway/internal/util" ) type HealthCheckConfig struct { - Interval time.Duration `yaml:"interval"` - Timeout time.Duration `yaml:"timeout"` - FailureThreshold uint `yaml:"failureThreshold"` - SuccessThreshold uint `yaml:"successThreshold"` + Interval util.DurationUnmarshalled `json:"interval"` + Timeout util.DurationUnmarshalled `json:"timeout"` + FailureThreshold uint `json:"failureThreshold"` + SuccessThreshold uint `json:"successThreshold"` } type ProxyConfig struct { // nolint:revive - Path string `yaml:"path"` - UpstreamTimeout time.Duration `yaml:"upstreamTimeout"` + Path string `json:"path"` + UpstreamTimeout util.DurationUnmarshalled `json:"upstreamTimeout"` } // This struct is temporary. It's about to keep the input interface clean and simple. diff --git a/internal/proxy/healthchecker.go b/internal/proxy/healthchecker.go index cb7cc25..fdb28ca 100644 --- a/internal/proxy/healthchecker.go +++ b/internal/proxy/healthchecker.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" ) @@ -21,16 +23,16 @@ type HealthCheckerConfig struct { Logger *slog.Logger // How often to check health. - Interval time.Duration `yaml:"healthcheckInterval"` + Interval util.DurationUnmarshalled `json:"interval"` // How long to wait for responses before failing - Timeout time.Duration `yaml:"healthcheckTimeout"` + Timeout util.DurationUnmarshalled `json:"timeout"` // Try FailureThreshold times before marking as unhealthy - FailureThreshold uint `yaml:"healthcheckInterval"` + FailureThreshold uint `yaml:"failureThreshold"` // Minimum consecutive successes required to mark as healthy - SuccessThreshold uint `yaml:"healthcheckInterval"` + SuccessThreshold uint `yaml:"successThreshold"` } type HealthChecker struct { @@ -93,6 +95,7 @@ func (h *HealthChecker) checkBlockNumber(c context.Context) (uint64, error) { // want to perform an eth_call to make sure eth_call requests are also succeding // as blockNumber can be either cached or routed to a different service on the // RPC provider's side. +// nolint: unused func (h *HealthChecker) checkGasLimit(c context.Context) (uint64, error) { gasLimit, err := performGasLeftCall(c, h.httpClient, h.config.URL) if err != nil { @@ -117,7 +120,7 @@ func (h *HealthChecker) CheckAndSetHealth() { } func (h *HealthChecker) checkAndSetBlockNumberHealth() { - c, cancel := context.WithTimeout(context.Background(), h.config.Timeout) + c, cancel := context.WithTimeout(context.Background(), time.Duration(h.config.Timeout)) defer cancel() // TODO @@ -135,8 +138,9 @@ func (h *HealthChecker) checkAndSetBlockNumberHealth() { h.blockNumber = blockNumber } +// nolint: unused func (h *HealthChecker) checkAndSetGasLeftHealth() { - c, cancel := context.WithTimeout(context.Background(), h.config.Timeout) + c, cancel := context.WithTimeout(context.Background(), time.Duration(h.config.Timeout)) defer cancel() gasLimit, err := h.checkGasLimit(c) @@ -154,7 +158,7 @@ func (h *HealthChecker) checkAndSetGasLeftHealth() { func (h *HealthChecker) Start(c context.Context) { h.CheckAndSetHealth() - ticker := time.NewTicker(h.config.Interval) + ticker := time.NewTicker(time.Duration(h.config.Interval)) defer ticker.Stop() for { diff --git a/internal/proxy/healthchecker_test.go b/internal/proxy/healthchecker_test.go index 4e46788..a638666 100644 --- a/internal/proxy/healthchecker_test.go +++ b/internal/proxy/healthchecker_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/caitlinelfring/go-env-default" "github.com/stretchr/testify/assert" ) @@ -18,8 +20,8 @@ func TestBasicHealthchecker(t *testing.T) { healtcheckConfig := HealthCheckerConfig{ URL: env.GetDefault("RPC_GATEWAY_NODE_URL_1", "https://cloudflare-eth.com"), - Interval: 1 * time.Second, - Timeout: 2 * time.Second, + Interval: util.DurationUnmarshalled(1 * time.Second), + Timeout: util.DurationUnmarshalled(2 * time.Second), FailureThreshold: 1, SuccessThreshold: 1, Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 19dbd44..7f404f4 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -23,7 +23,7 @@ type Proxy struct { func NewProxy(config Config) (*Proxy, error) { proxy := &Proxy{ hcm: config.HealthcheckManager, - timeout: config.Proxy.UpstreamTimeout, + timeout: time.Duration(config.Proxy.UpstreamTimeout), metricRequestDuration: promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: "zeroex_rpc_gateway_request_duration_seconds_" + config.Name, diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 910d5b7..05f2b55 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/go-http-utils/headers" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" @@ -20,7 +22,7 @@ import ( func createConfig() Config { return Config{ Proxy: ProxyConfig{ - UpstreamTimeout: time.Second * 3, + UpstreamTimeout: util.DurationUnmarshalled(time.Second * 3), }, HealthChecks: HealthCheckConfig{ Interval: 0, diff --git a/internal/rpcgateway/config.go b/internal/rpcgateway/config.go index d892743..30a97fb 100644 --- a/internal/rpcgateway/config.go +++ b/internal/rpcgateway/config.go @@ -6,9 +6,9 @@ import ( ) type RPCGatewayConfig struct { //nolint:revive - Name string `yaml:"name"` - Metrics metrics.Config `yaml:"metrics"` - Proxy proxy.ProxyConfig `yaml:"proxy"` - HealthChecks proxy.HealthCheckConfig `yaml:"healthChecks"` - Targets []proxy.NodeProviderConfig `yaml:"targets"` + Name string `json:"name"` + Metrics metrics.Config `json:"metrics"` + Proxy proxy.ProxyConfig `json:"proxy"` + HealthChecks proxy.HealthCheckConfig `json:"healthChecks"` + Targets []proxy.NodeProviderConfig `json:"targets"` } diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 29ae0bd..cbeb9c9 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -79,8 +79,8 @@ func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux) (*RPCGateway, error // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. -func NewRPCGatewayFromConfigFile(fileOrURL string, router *chi.Mux) (*RPCGateway, error) { - config, err := util.LoadYamlFile[RPCGatewayConfig](fileOrURL) +func NewRPCGatewayFromConfigFile(configFile string, router *chi.Mux) (*RPCGateway, error) { + config, err := util.LoadJSONFile[RPCGatewayConfig](configFile) if err != nil { return nil, errors.Wrap(err, "failed to load config") } diff --git a/internal/util/util.go b/internal/util/util.go index ed574b8..cb414c8 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -1,44 +1,52 @@ package util import ( + "encoding/json" "errors" - "fmt" "io" "net/http" "net/url" "os" - - "gopkg.in/yaml.v2" + "time" ) -// LoadYamlFile is refactored to use a generic type T. -// T must be a type that can be unmarshaled from JSON. -func LoadYamlFile[T any](pathOrURL string) (*T, error) { +// LoadJSONFile attempts to load and parse a JSON file into a Go struct. The input can be a filepath, +// a URL, or an environment variable name containing the JSON content. +func LoadJSONFile[T any](file string) (*T, error) { var data []byte var err error - fmt.Printf("Loading gateways configuration from: %s", pathOrURL) - if isValidURL(pathOrURL) { - data, err = loadFileFromURL(pathOrURL) - if err != nil { - return nil, err - } + // Check if file is an environment variable containing JSON data + if raw, isInENV := os.LookupEnv(file); isInENV { + data = []byte(raw) } else { - data, err = os.ReadFile(pathOrURL) + // Load data from URL or local file + if IsValidURL(file) { + data, err = ReadFileFromURL(file) + } else { + data, err = os.ReadFile(file) + } if err != nil { return nil, err } } + // Parse JSON data into the specified struct type + return ParseJSONlFile[T](data) +} + +// ParseJSONlFile parses JSON data into a struct of type T. +func ParseJSONlFile[T any](data []byte) (*T, error) { var config T - if err := yaml.Unmarshal(data, &config); err != nil { + if err := json.Unmarshal(data, &config); err != nil { return nil, err } return &config, nil } -func loadFileFromURL(pathOrURL string) ([]byte, error) { +// ReadFileFromURL fetches the content of a file from a URL. +func ReadFileFromURL(pathOrURL string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, pathOrURL, nil) if err != nil { return nil, err @@ -50,18 +58,14 @@ func loadFileFromURL(pathOrURL string) ([]byte, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, errors.New("failed to fetch config from URL") + return nil, errors.New("failed to fetch config from URL: status code " + resp.Status) } - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - return data, nil + return io.ReadAll(resp.Body) } -func isValidURL(toTest string) bool { +// IsValidURL checks if the given string is a well-formed URL. +func IsValidURL(toTest string) bool { u, err := url.Parse(toTest) if err != nil { return false @@ -69,3 +73,29 @@ func isValidURL(toTest string) bool { return u.Scheme != "" && u.Host != "" } + +// DurationUnmarshalled is a wrapper around time.Duration to handle JSON unmarshalling. +type DurationUnmarshalled time.Duration + +// UnmarshalJSON converts a JSON string to a DurationUnmarshalled. +func (d *DurationUnmarshalled) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + *d = DurationUnmarshalled(time.Duration(value)) + case string: + var err error + duration, err := time.ParseDuration(value) + if err != nil { + return err + } + *d = DurationUnmarshalled(duration) + default: + return errors.New("invalid duration") + } + + return nil +} diff --git a/main.go b/main.go index 3ce4f3a..a9c28f3 100644 --- a/main.go +++ b/main.go @@ -23,19 +23,21 @@ import ( "github.com/urfave/cli/v2" ) -type MetricsConfig struct { - Port int `yaml:"port"` +// Config represents the application configuration structure, +// including metrics and gateway configurations. +type Config struct { + Metrics MetricsConfig `json:"metrics"` + Port uint `json:"port"` + Gateways []GatewayConfig `json:"gateways"` } -type Config struct { - Metrics MetricsConfig `yaml:"metrics"` - Port string `yaml:"port"` - Gateways []GatewayConfig `yaml:"gateways"` +type MetricsConfig struct { + Port uint `json:"port"` } type GatewayConfig struct { - ConfigFile string `yaml:"configFile"` - Name string `yaml:"name"` + ConfigFile string `json:"configFile"` + Name string `json:"name"` } func main() { @@ -48,48 +50,31 @@ func main() { Flags: []cli.Flag{ &cli.StringFlag{ Name: "config", - Usage: "The YAML configuration file path with gateway configurations.", - Value: "config.yaml", // Default configuration file name + Usage: "The JSON configuration file path with gateway configurations.", + Value: "config.JSON", // Default configuration file name + }, + &cli.BoolFlag{ + Name: "env", + Usage: "Load configuration from environment variable named GATEWAY_CONFIG.", + Value: false, }, }, Action: func(cc *cli.Context) error { - configPath := cc.String("config") - config, err := util.LoadYamlFile[Config](configPath) + configPath := resolveConfigPath(cc.String("config"), cc.Bool("env")) + config, err := util.LoadJSONFile[Config](configPath) if err != nil { return errors.Wrap(err, "failed to load config") } - logLevel := slog.LevelWarn - if os.Getenv("DEBUG") == "true" { - logLevel = slog.LevelDebug - } - - logger := httplog.NewLogger("rpc-gateway", httplog.Options{ - JSON: true, - RequestHeaders: true, - LogLevel: logLevel, - }) - - metricsServer := metrics.NewServer(metrics.Config{Port: uint(config.Metrics.Port)}) - go func() { - err = metricsServer.Start() - defer func(metricsServer *metrics.Server) { - err := metricsServer.Stop() - if err != nil { - fmt.Fprintf(os.Stderr, "error stopping metrics server: %v\n", err) - } - }(metricsServer) - if err != nil { - fmt.Fprintf(os.Stderr, "error starting metrics server: %v\n", err) - } - }() + logger := configureLogger() + startMetricsServer(config.Metrics.Port) r := chi.NewRouter() r.Use(httplog.RequestLogger(logger)) r.Use(middleware.Recoverer) r.Use(middleware.Heartbeat("/health")) server := &http.Server{ - Addr: fmt.Sprintf(":%s", config.Port), + Addr: fmt.Sprintf(":%d", config.Port), Handler: r, WriteTimeout: time.Second * 15, ReadTimeout: time.Second * 15, @@ -109,7 +94,7 @@ func main() { }(gatewayConfig) } - fmt.Println("Starting RPC Gateway server on port: " + config.Port) + fmt.Printf("Starting RPC Gateway server on port: %d\n", config.Port) err = server.ListenAndServe() if err != nil { return err @@ -126,6 +111,43 @@ func main() { } } +func resolveConfigPath(config string, isENV bool) string { + if isENV { + return "GATEWAY_CONFIG" + } + + return config +} + +func configureLogger() *httplog.Logger { + logLevel := slog.LevelWarn + if os.Getenv("DEBUG") == "true" { + logLevel = slog.LevelDebug + } + + return httplog.NewLogger("rpc-gateway", httplog.Options{ + JSON: true, + RequestHeaders: true, + LogLevel: logLevel, + }) +} + +func startMetricsServer(port uint) { + metricsServer := metrics.NewServer(metrics.Config{Port: port}) + go func() { + err := metricsServer.Start() + defer func(metricsServer *metrics.Server) { + err := metricsServer.Stop() + if err != nil { + fmt.Fprintf(os.Stderr, "error stopping metrics server: %v\n", err) + } + }(metricsServer) + if err != nil { + fmt.Fprintf(os.Stderr, "error starting metrics server: %v\n", err) + } + }() +} + func startGateway(ctx context.Context, config GatewayConfig, router *chi.Mux) error { service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, router) if err != nil {