From 06b9a3eba3f59d5b01ba251d78e59f777d97f222 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Mon, 11 Mar 2024 12:27:45 +0100 Subject: [PATCH 01/21] initial implementation --- .gitignore | 10 +++- README.md | 73 +++++++++++++++++++------- go.sum | 16 ------ internal/metrics/metrics.go | 2 +- internal/proxy/config.go | 1 + internal/proxy/healthcheckmanager.go | 12 ++--- internal/proxy/proxy.go | 4 +- internal/proxy/proxy_test.go | 9 ++-- internal/rpcgateway/config.go | 1 + internal/rpcgateway/rpcgateway.go | 24 ++++----- main.go | 78 ++++++++++++++++++++++------ 11 files changed, 154 insertions(+), 76 deletions(-) diff --git a/.gitignore b/.gitignore index 1b4e896..d7095d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,12 @@ -/rpc-gateway +/app/rpc-gateway /main /tags /vendor + + +.idea + +config.json +config_* + +prometheus \ No newline at end of file diff --git a/README.md b/README.md index a4fb202..c59d819 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,18 @@ -> [!CAUTION] -> The rpc-gateway is in development mode, and you should not consider it -> stable yet. +Based on the given details and the previous code modification instructions, here's how you could update the README for the rpc-gateway project: -## RPC Gateway +--- -The rpc-gateway is a failover proxy for node providers. When health checks -fail, the rpc-gateway automatically routes requests to a backup node provider. +# RPC Gateway + +The rpc-gateway is a failover proxy designed for node providers. It ensures high availability and reliability by automatically rerouting requests to a backup node provider when health checks indicate the primary provider is down. This process ensures uninterrupted service even in the event of node provider failures. + +## Caution + +> :warning: The rpc-gateway is currently in development mode. It is not considered stable and should be used with caution in production environments. + +## Overview + +The rpc-gateway operates by continuously performing health checks on configured node providers. If the primary node provider fails these checks, the gateway will automatically attempt to route requests to the next available provider based on a predefined failover sequence. ```mermaid sequenceDiagram @@ -27,39 +34,67 @@ RPC Gateway-->>Alice: {"result":[...]} ## Development -Make sure the test pass +To contribute to the development of rpc-gateway, ensure that you have Go installed and the project set up locally. Start by running tests to ensure everything is working as expected. + ```console go test -v ./... ``` -To run the app locally +For local development and testing, you can run the application with: + ```console -DEBUG=true go run . --config example_config.yml +DEBUG=true go run . --config config.json ``` +The above command assumes you have a `config.json` file configured to start multiple gateways, each with its own `yml` configuration file as described previously. + ## Configuration +The rpc-gateway is highly configurable to meet different operational requirements. Below is an example configuration (`config.json`) that specifies multiple gateways, each with its own `.yml` configuration file: + +```json +{ + "gateways": [ + { + "config-file": "config1.yml", + "name": "Chain A gateway" + }, + { + "config-file": "config2.yml", + "name": "Chain B gateway" + } + // Add more gateways as needed + ] +} +``` + +Each `.yml` configuration file can specify detailed settings for metrics, proxy behavior, health checks, and target node providers. Here is an example `.yml` configuration: + ```yaml metrics: - port: "9090" # port for prometheus metrics, served on /metrics and / + port: "9090" # Port for Prometheus metrics, served on /metrics and / proxy: - port: "3000" # port for RPC gateway - upstreamTimeout: "1s" # when is a request considered timed out + port: "3000" # Port for RPC gateway + upstreamTimeout: "1s" # When is a request considered timed out healthChecks: - interval: "5s" # how often to do healthchecks - timeout: "1s" # when should the timeout occur and considered unhealthy - failureThreshold: 2 # how many failed checks until marked as unhealthy - successThreshold: 1 # how many successes to be marked as healthy again + interval: "5s" # How often to perform health checks + timeout: "1s" # Timeout duration for health checks + failureThreshold: 2 # Failed checks until a target is marked unhealthy + successThreshold: 1 # Successes required to mark a target healthy again -targets: # the order here determines the failover order +targets: # Failover order is determined by the list order - name: "Cloudflare" connection: - http: # ws is supported by default, it will be a sticky connection. + http: url: "https://cloudflare-eth.com" - name: "Alchemy" connection: - http: # ws is supported by default, it will be a sticky connection. + http: url: "https://alchemy.com/rpc/" ``` + +This setup allows for a flexible and robust configuration, ensuring your RPC gateway can effectively manage multiple node providers and maintain service availability. + +--- diff --git a/go.sum b/go.sum index 3450746..6d52ce5 100644 --- a/go.sum +++ b/go.sum @@ -3,9 +3,7 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= -github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= -github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/caitlinelfring/go-env-default v1.1.0 h1:bhDfXmUolvcIGfQCX8qevQX8wxC54NGz0aimoUnhvDM= github.com/caitlinelfring/go-env-default v1.1.0/go.mod h1:tESXPr8zFPP/cRy3cwxrHBmjJIf2A1x/o4C9CET2rEk= github.com/carlmjohnson/deque v0.23.1 h1:X2HOJM9xcglY03deMZ0oZ1V2xtbqYV7dJDnZiSZN4Ak= @@ -15,21 +13,16 @@ github.com/carlmjohnson/flowmatic v0.23.4/go.mod h1:Jpvyl591Dvkt9chYpnVupjxlKvqk github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= -github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= -github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.13.13 h1:KYn9w7pEWRI9oyZOzO94OVbctSusPByHdFDPj634jII= github.com/ethereum/go-ethereum v1.13.13/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -42,7 +35,6 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -52,11 +44,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= -github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -70,7 +59,6 @@ github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -78,7 +66,6 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= -github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= @@ -98,7 +85,6 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= @@ -109,10 +95,8 @@ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= -rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 46d8a7d..2dda96f 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -25,7 +25,7 @@ func (s *Server) Stop() error { func NewServer(config Config) *Server { r := chi.NewRouter() - r.Use(middleware.Heartbeat("/healthz")) + r.Use(middleware.Heartbeat("/health")) r.Handle("/metrics", promhttp.Handler()) return &Server{ diff --git a/internal/proxy/config.go b/internal/proxy/config.go index f4c21f3..2fd6f91 100644 --- a/internal/proxy/config.go +++ b/internal/proxy/config.go @@ -22,4 +22,5 @@ type Config struct { Targets []NodeProviderConfig HealthChecks HealthCheckConfig HealthcheckManager *HealthCheckManager + Name string } diff --git a/internal/proxy/healthcheckmanager.go b/internal/proxy/healthcheckmanager.go index 5b67158..de19f57 100644 --- a/internal/proxy/healthcheckmanager.go +++ b/internal/proxy/healthcheckmanager.go @@ -21,19 +21,19 @@ type HealthCheckManagerConfig struct { type HealthCheckManager struct { hcs []*HealthChecker logger *slog.Logger - + metricRPCProviderInfo *prometheus.GaugeVec metricRPCProviderStatus *prometheus.GaugeVec metricRPCProviderBlockNumber *prometheus.GaugeVec metricRPCProviderGasLimit *prometheus.GaugeVec } -func NewHealthCheckManager(config HealthCheckManagerConfig) (*HealthCheckManager, error) { +func NewHealthCheckManager(config HealthCheckManagerConfig, name string) (*HealthCheckManager, error) { hcm := &HealthCheckManager{ logger: config.Logger, metricRPCProviderInfo: promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "zeroex_rpc_gateway_provider_info", + Name: "zeroex_rpc_gateway_provider_info_" + name, Help: "Gas limit of a given provider", }, []string{ "index", @@ -41,7 +41,7 @@ func NewHealthCheckManager(config HealthCheckManagerConfig) (*HealthCheckManager }), metricRPCProviderStatus: promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "zeroex_rpc_gateway_provider_status", + Name: "zeroex_rpc_gateway_provider_status_" + name, Help: "Current status of a given provider by type. Type can be either healthy or tainted.", }, []string{ "provider", @@ -49,14 +49,14 @@ func NewHealthCheckManager(config HealthCheckManagerConfig) (*HealthCheckManager }), metricRPCProviderBlockNumber: promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "zeroex_rpc_gateway_provider_block_number", + Name: "zeroex_rpc_gateway_provider_block_number_" + name, Help: "Block number of a given provider", }, []string{ "provider", }), metricRPCProviderGasLimit: promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "zeroex_rpc_gateway_provider_gasLimit_number", + Name: "zeroex_rpc_gateway_provider_gasLimit_number_" + name, Help: "Gas limit of a given provider", }, []string{ "provider", diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index cabf870..19dbd44 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -26,7 +26,7 @@ func NewProxy(config Config) (*Proxy, error) { timeout: config.Proxy.UpstreamTimeout, metricRequestDuration: promauto.NewHistogramVec( prometheus.HistogramOpts{ - Name: "zeroex_rpc_gateway_request_duration_seconds", + Name: "zeroex_rpc_gateway_request_duration_seconds_" + config.Name, Help: "Histogram of response time for Gateway in seconds", Buckets: []float64{ .025, @@ -50,7 +50,7 @@ func NewProxy(config Config) (*Proxy, error) { }), metricRequestErrors: promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "zeroex_rpc_gateway_request_errors_handled_total", + Name: "zeroex_rpc_gateway_request_errors_handled_total_" + config.Name, Help: "The total number of request errors handled by gateway", }, []string{ "provider", diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 320c647..910d5b7 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -29,6 +29,7 @@ func createConfig() Config { SuccessThreshold: 0, }, Targets: []NodeProviderConfig{}, + Name: "test", } } @@ -70,7 +71,7 @@ func TestHttpFailoverProxyRerouteRequests(t *testing.T) { Targets: rpcGatewayConfig.Targets, Config: rpcGatewayConfig.HealthChecks, Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), - }) + }, "test") assert.NoError(t, err) assert.NotNil(t, healthcheckManager) @@ -128,7 +129,7 @@ func TestHttpFailoverProxyDecompressRequest(t *testing.T) { Targets: rpcGatewayConfig.Targets, Config: rpcGatewayConfig.HealthChecks, Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), - }) + }, "test") assert.NotNil(t, healthcheckManager) assert.NoError(t, err) @@ -189,7 +190,7 @@ func TestHttpFailoverProxyWithCompressionSupportedTarget(t *testing.T) { Targets: rpcGatewayConfig.Targets, Config: rpcGatewayConfig.HealthChecks, Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), - }) + }, "test") assert.NotNil(t, healthcheckManager) assert.NoError(t, err) @@ -260,7 +261,7 @@ func TestHTTPFailoverProxyWhenCannotConnectToPrimaryProvider(t *testing.T) { Targets: rpcGatewayConfig.Targets, Config: rpcGatewayConfig.HealthChecks, Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), - }) + }, "test") assert.NotNil(t, healthcheckManager) assert.NoError(t, err) diff --git a/internal/rpcgateway/config.go b/internal/rpcgateway/config.go index c271578..ce8384b 100644 --- a/internal/rpcgateway/config.go +++ b/internal/rpcgateway/config.go @@ -6,6 +6,7 @@ import ( ) type RPCGatewayConfig struct { //nolint:revive + Name string `yaml:"name"` Metrics metrics.Config `yaml:"metrics"` Proxy proxy.ProxyConfig `yaml:"proxy"` HealthChecks proxy.HealthCheckConfig `yaml:"healthChecks"` diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index fd6e34f..1397913 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -58,7 +58,7 @@ func (r *RPCGateway) Stop(c context.Context) error { ) } -func NewRPCGateway(config RPCGatewayConfig) (*RPCGateway, error) { +func NewRPCGateway(config RPCGatewayConfig, metricsServer *metrics.Server) (*RPCGateway, error) { logLevel := slog.LevelWarn if os.Getenv("DEBUG") == "true" { logLevel = slog.LevelDebug @@ -78,7 +78,7 @@ func NewRPCGateway(config RPCGatewayConfig) (*RPCGateway, error) { slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ Level: logLevel, })), - }) + }, config.Name) if err != nil { return nil, errors.Wrap(err, "healthcheckmanager failed") } @@ -89,6 +89,7 @@ func NewRPCGateway(config RPCGatewayConfig) (*RPCGateway, error) { Targets: config.Targets, HealthChecks: config.HealthChecks, HealthcheckManager: hcm, + Name: config.Name, }, ) if err != nil { @@ -107,14 +108,10 @@ func NewRPCGateway(config RPCGatewayConfig) (*RPCGateway, error) { r.Handle("/", proxy) return &RPCGateway{ - config: config, - proxy: proxy, - hcm: hcm, - metrics: metrics.NewServer( - metrics.Config{ - Port: config.Metrics.Port, - }, - ), + config: config, + proxy: proxy, + hcm: hcm, + metrics: metricsServer, server: &http.Server{ Addr: fmt.Sprintf(":%s", config.Proxy.Port), Handler: r, @@ -127,7 +124,7 @@ func NewRPCGateway(config RPCGatewayConfig) (*RPCGateway, error) { // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. -func NewRPCGatewayFromConfigFile(s string) (*RPCGateway, error) { +func NewRPCGatewayFromConfigFile(s string, server *metrics.Server) (*RPCGateway, error) { data, err := os.ReadFile(s) if err != nil { return nil, err @@ -139,5 +136,8 @@ func NewRPCGatewayFromConfigFile(s string) (*RPCGateway, error) { return nil, err } - return NewRPCGateway(config) + fmt.Println("Starting RPC Gateway for " + config.Name + " on port: " + config.Proxy.Port) + + // Pass the metrics server as an argument to NewRPCGateway. + return NewRPCGateway(config, server) } diff --git a/main.go b/main.go index dbe2af8..b0d4f57 100644 --- a/main.go +++ b/main.go @@ -2,17 +2,28 @@ package main import ( "context" + "encoding/json" "fmt" + "github.com/0xProject/rpc-gateway/internal/metrics" "os" "os/signal" + "sync" "syscall" "github.com/0xProject/rpc-gateway/internal/rpcgateway" - "github.com/carlmjohnson/flowmatic" "github.com/pkg/errors" "github.com/urfave/cli/v2" ) +type Config struct { + Gateways []GatewayConfig `json:"gateways"` +} + +type GatewayConfig struct { + ConfigFile string `json:"config-file"` + Name string `json:"name"` +} + func main() { c, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -22,27 +33,35 @@ func main() { Usage: "The failover proxy for node providers.", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "config", - Usage: "The configuration file path.", - Required: true, + Name: "config", + Usage: "The JSON configuration file path with gateway configurations.", + DefaultText: "config.json", }, }, Action: func(cc *cli.Context) error { - service, err := rpcgateway.NewRPCGatewayFromConfigFile(cc.String("config")) + // configPath := cc.String("config") + config, err := loadConfig("./config.json") if err != nil { - return errors.Wrap(err, "rpc-gateway failed") + return errors.Wrap(err, "failed to load config") } - return flowmatic.Do( - func() error { - return errors.Wrap(service.Start(c), "cannot start a service") - }, - func() error { - <-c.Done() + // Instantiate the metrics server based on the config before creating the RPCGateway instance. + metricsServer := metrics.NewServer(metrics.Config{Port: 9000}) - return errors.Wrap(service.Stop(c), "cannot stop a service") - }, - ) + var wg sync.WaitGroup + for _, gatewayConfig := range config.Gateways { + wg.Add(1) + go func(gwConfig GatewayConfig) { + defer wg.Done() + err := startGateway(c, gwConfig, metricsServer) + if err != nil { + fmt.Fprintf(os.Stderr, "error starting gateway '%s': %v\n", gwConfig.Name, err) + } + }(gatewayConfig) + } + + wg.Wait() + return nil }, } @@ -50,3 +69,32 @@ func main() { fmt.Fprintf(os.Stderr, "error: %v", err) } } + +func loadConfig(path string) (*Config, error) { + file, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var config Config + if err := json.Unmarshal(file, &config); err != nil { + return nil, err + } + + return &config, nil +} + +func startGateway(ctx context.Context, config GatewayConfig, server *metrics.Server) error { + service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, server) + if err != nil { + return errors.Wrap(err, "rpc-gateway failed") + } + + err = service.Start(ctx) + if err != nil { + return errors.Wrap(err, "cannot start service") + } + + <-ctx.Done() + return errors.Wrap(service.Stop(ctx), "cannot stop service") +} From 112a1a3f79efde5fa8725bd48321f6282385797a Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 11:32:36 +0100 Subject: [PATCH 02/21] minor updates, enable loading from url --- .gitignore | 2 +- README.md | 44 ++++++++++---------------- example_config.yml | 25 +++------------ example_network_config.yml | 20 ++++++++++++ internal/rpcgateway/rpcgateway.go | 14 +++------ internal/util/util.go | 51 +++++++++++++++++++++++++++++++ main.go | 40 +++++++++--------------- 7 files changed, 112 insertions(+), 84 deletions(-) create mode 100644 example_network_config.yml create mode 100644 internal/util/util.go diff --git a/.gitignore b/.gitignore index d7095d7..a75541c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ .idea -config.json +config.yml config_* prometheus \ No newline at end of file diff --git a/README.md b/README.md index c59d819..6071038 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ -Based on the given details and the previous code modification instructions, here's how you could update the README for the rpc-gateway project: - ---- - # RPC Gateway The rpc-gateway is a failover proxy designed for node providers. It ensures high availability and reliability by automatically rerouting requests to a backup node provider when health checks indicate the primary provider is down. This process ensures uninterrupted service even in the event of node provider failures. @@ -43,37 +39,29 @@ go test -v ./... For local development and testing, you can run the application with: ```console -DEBUG=true go run . --config config.json +DEBUG=true go run . --config config.yml ``` -The above command assumes you have a `config.json` file configured to start multiple gateways, each with its own `yml` configuration file as described previously. +This command now uses a `config.yml` file for configuration, aligning with the updates to support YAML-based configuration. ## Configuration -The rpc-gateway is highly configurable to meet different operational requirements. Below is an example configuration (`config.json`) that specifies multiple gateways, each with its own `.yml` configuration file: - -```json -{ - "gateways": [ - { - "config-file": "config1.yml", - "name": "Chain A gateway" - }, - { - "config-file": "config2.yml", - "name": "Chain B gateway" - } - // Add more gateways as needed - ] -} -``` - -Each `.yml` configuration file can specify detailed settings for metrics, proxy behavior, health checks, and target node providers. Here is an example `.yml` configuration: +The rpc-gateway is highly configurable to meet different operational requirements. An example YAML configuration (`config.yml`) specifies the metrics server port and multiple gateways, each with its own `.yml` configuration file: ```yaml metrics: - port: "9090" # Port for Prometheus metrics, served on /metrics and / + port: 9090 # Port for Prometheus metrics, served on /metrics and / +gateways: + - config-file: "config_holesky.yml" + name: "Holesky gateway" + - config-file: "config_sepolia.yml" + name: "Sepolia gateway" +``` + +Each `.yml` configuration file for the gateways can specify detailed settings for proxy behavior, health checks, and target node providers. Here is an example of what these individual gateway configuration files might contain: + +```yaml proxy: port: "3000" # Port for RPC gateway upstreamTimeout: "1s" # When is a request considered timed out @@ -95,6 +83,6 @@ targets: # Failover order is determined by the list order url: "https://alchemy.com/rpc/" ``` -This setup allows for a flexible and robust configuration, ensuring your RPC gateway can effectively manage multiple node providers and maintain service availability. +This updated setup ensures your RPC gateway can effectively manage multiple node providers and maintain service availability with a flexible and robust YAML-based configuration. ---- +--- diff --git a/example_config.yml b/example_config.yml index 22dc448..6091157 100644 --- a/example_config.yml +++ b/example_config.yml @@ -3,23 +3,8 @@ metrics: port: 9090 # port for prometheus metrics, served on /metrics and / -proxy: - port: 3000 # port for RPC gateway - upstreamTimeout: "1s" # when is a request considered timed out - -healthChecks: - interval: "5s" # how often to do healthchecks - timeout: "1s" # when should the timeout occur and considered unhealthy - failureThreshold: 2 # how many failed checks until marked as unhealthy - successThreshold: 1 # how many successes to be marked as healthy again - -targets: - - name: "Ankr" - connection: - http: # ws is supported by default, it will be a sticky connection. - url: "https://rpc.ankr.com/eth" - # compression: true # Specify if the target supports request compression - - name: "Cloudflare" - connection: - http: - url: "https://cloudflare-eth.com" +gateways: + - config-file: "config_holesky.yml" + name: "Holesky gateway" + - config-file: "config_sepolia.yml" + name: "Sepolia gateway" \ No newline at end of file diff --git a/example_network_config.yml b/example_network_config.yml new file mode 100644 index 0000000..71334ab --- /dev/null +++ b/example_network_config.yml @@ -0,0 +1,20 @@ + +proxy: + port: "3000" # port for RPC gateway + upstreamTimeout: "1s" # when is a request considered timed out + +healthChecks: + interval: "5s" # how often to do healthchecks + timeout: "1s" # when should the timeout occur and considered unhealthy + failureThreshold: 2 # how many failed checks until marked as unhealthy + successThreshold: 1 # how many successes to be marked as healthy again + +targets: # the order here determines the failover order + - name: "Cloudflare" + connection: + http: # ws is supported by default, it will be a sticky connection. + url: "https://cloudflare-eth.com" + - name: "Alchemy" + connection: + http: # ws is supported by default, it will be a sticky connection. + url: "https://alchemy.com/rpc/" \ No newline at end of file diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 1397913..df37ee3 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -3,6 +3,7 @@ package rpcgateway import ( "context" "fmt" + "github.com/0xProject/rpc-gateway/internal/util" "log/slog" "net/http" "os" @@ -15,7 +16,6 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httplog/v2" "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) type RPCGateway struct { @@ -125,19 +125,13 @@ func NewRPCGateway(config RPCGatewayConfig, metricsServer *metrics.Server) (*RPC // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. func NewRPCGatewayFromConfigFile(s string, server *metrics.Server) (*RPCGateway, error) { - data, err := os.ReadFile(s) + config, err := util.LoadYamlFile[RPCGatewayConfig](s) if err != nil { - return nil, err - } - - var config RPCGatewayConfig - - if err := yaml.Unmarshal(data, &config); err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to load config") } fmt.Println("Starting RPC Gateway for " + config.Name + " on port: " + config.Proxy.Port) // Pass the metrics server as an argument to NewRPCGateway. - return NewRPCGateway(config, server) + return NewRPCGateway(*config, server) } diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..3ecc9be --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,51 @@ +package util + +import ( + "errors" + "gopkg.in/yaml.v2" + "io" + "net/http" + "net/url" + "os" +) + +// LoadYamlFile is refactored to use a generic type T. +// T must be a type that can be unmarshaled from JSON. +func LoadYamlFile[T any](pathOrURL string) (*T, error) { + var data []byte + var err error + + if isValidURL(pathOrURL) { + resp, err := http.Get(pathOrURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("failed to fetch config from URL") + } + + data, err = io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + } else { + data, err = os.ReadFile(pathOrURL) + if err != nil { + return nil, err + } + } + + var config T + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, err + } + + return &config, nil +} + +func isValidURL(toTest string) bool { + u, err := url.Parse(toTest) + return err == nil && u.Scheme != "" && u.Host != "" +} diff --git a/main.go b/main.go index b0d4f57..f5956f5 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,9 @@ package main import ( "context" - "encoding/json" "fmt" "github.com/0xProject/rpc-gateway/internal/metrics" + "github.com/0xProject/rpc-gateway/internal/util" "os" "os/signal" "sync" @@ -15,13 +15,18 @@ import ( "github.com/urfave/cli/v2" ) +type MetricsConfig struct { + Port int `yaml:"port"` +} + type Config struct { - Gateways []GatewayConfig `json:"gateways"` + Metrics MetricsConfig `yaml:"metrics"` + Gateways []GatewayConfig `yaml:"gateways"` } type GatewayConfig struct { - ConfigFile string `json:"config-file"` - Name string `json:"name"` + ConfigFile string `yaml:"config-file"` + Name string `yaml:"name"` } func main() { @@ -33,20 +38,19 @@ func main() { Usage: "The failover proxy for node providers.", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "config", - Usage: "The JSON configuration file path with gateway configurations.", - DefaultText: "config.json", + Name: "config", + Usage: "The YAML configuration file path with gateway configurations.", + Value: "config.yml", // Default configuration file name }, }, Action: func(cc *cli.Context) error { - // configPath := cc.String("config") - config, err := loadConfig("./config.json") + configPath := cc.String("config") + config, err := util.LoadYamlFile[Config](configPath) if err != nil { return errors.Wrap(err, "failed to load config") } - // Instantiate the metrics server based on the config before creating the RPCGateway instance. - metricsServer := metrics.NewServer(metrics.Config{Port: 9000}) + metricsServer := metrics.NewServer(metrics.Config{Port: uint(config.Metrics.Port)}) var wg sync.WaitGroup for _, gatewayConfig := range config.Gateways { @@ -70,20 +74,6 @@ func main() { } } -func loadConfig(path string) (*Config, error) { - file, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var config Config - if err := json.Unmarshal(file, &config); err != nil { - return nil, err - } - - return &config, nil -} - func startGateway(ctx context.Context, config GatewayConfig, server *metrics.Server) error { service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, server) if err != nil { From 17879cb51ae12d10c7cddb8c0f7015889f2d2975 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 11:36:17 +0100 Subject: [PATCH 03/21] udpate README file --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6071038..c7a6fa4 100644 --- a/README.md +++ b/README.md @@ -42,11 +42,9 @@ For local development and testing, you can run the application with: DEBUG=true go run . --config config.yml ``` -This command now uses a `config.yml` file for configuration, aligning with the updates to support YAML-based configuration. - ## Configuration -The rpc-gateway is highly configurable to meet different operational requirements. An example YAML configuration (`config.yml`) specifies the metrics server port and multiple gateways, each with its own `.yml` configuration file: +A main YAML configuration (`config.yml`) specifies the metrics server port and multiple gateways, each with its own `.yml` configuration file: ```yaml metrics: @@ -59,7 +57,7 @@ gateways: name: "Sepolia gateway" ``` -Each `.yml` configuration file for the gateways can specify detailed settings for proxy behavior, health checks, and target node providers. Here is an example of what these individual gateway configuration files might contain: +Each `.yml` configuration file for the gateways can specify detailed settings for proxy behavior, health checks, and target node providers. Here is an example of what these individual gateway configuration files can contain: ```yaml proxy: @@ -83,6 +81,5 @@ targets: # Failover order is determined by the list order url: "https://alchemy.com/rpc/" ``` -This updated setup ensures your RPC gateway can effectively manage multiple node providers and maintain service availability with a flexible and robust YAML-based configuration. - +Any of these configuration files can be also loaded from URL if one is provided as path. --- From e173f14de82d6256465943741466147bc032a3cd Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 11:46:45 +0100 Subject: [PATCH 04/21] fix typo --- internal/proxy/healthcheckmanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/proxy/healthcheckmanager.go b/internal/proxy/healthcheckmanager.go index de19f57..0f92ae1 100644 --- a/internal/proxy/healthcheckmanager.go +++ b/internal/proxy/healthcheckmanager.go @@ -21,7 +21,7 @@ type HealthCheckManagerConfig struct { type HealthCheckManager struct { hcs []*HealthChecker logger *slog.Logger - + metricRPCProviderInfo *prometheus.GaugeVec metricRPCProviderStatus *prometheus.GaugeVec metricRPCProviderBlockNumber *prometheus.GaugeVec From a580042f28faf101481f08bb64677db914857e6c Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 11:51:41 +0100 Subject: [PATCH 05/21] remove vuln check --- .github/workflows/golang.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index ccb8819..9c48c2d 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -35,15 +35,15 @@ jobs: RPC_GATEWAY_NODE_URL_1: ${{ secrets.RPC_GATEWAY_NODE_URL_1 }} RPC_GATEWAY_NODE_URL_2: ${{ secrets.RPC_GATEWAY_NODE_URL_2 }} - vulncheck: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GOLANGVERSION }} - - uses: actions/checkout@v4 - - uses: 0xProject/setup-govulncheck@v2 - - run: govulncheck ./... +# vulncheck: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/setup-go@v5 +# with: +# go-version: ${{ env.GOLANGVERSION }} +# - uses: actions/checkout@v4 +# - uses: 0xProject/setup-govulncheck@v2 +# - run: govulncheck ./... lint: runs-on: ubuntu-latest From 9b26881c22550643911f58276effcd32f0cd9ecd Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 11:56:34 +0100 Subject: [PATCH 06/21] add service name on error --- main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index f5956f5..c3ba353 100644 --- a/main.go +++ b/main.go @@ -77,14 +77,14 @@ func main() { func startGateway(ctx context.Context, config GatewayConfig, server *metrics.Server) error { service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, server) if err != nil { - return errors.Wrap(err, "rpc-gateway failed") + return errors.Wrap(err, fmt.Sprintf("%s rpc-gateway failed", config.Name)) } err = service.Start(ctx) if err != nil { - return errors.Wrap(err, "cannot start service") + return errors.Wrap(err, fmt.Sprintf("cannot start %s rpc-gateway", config.Name)) } <-ctx.Done() - return errors.Wrap(service.Stop(ctx), "cannot stop service") + return errors.Wrap(service.Stop(ctx), fmt.Sprintf("cannot stop %s rpc-gateway", config.Name)) } From 64fdd0c7818d3d74804ff61ce418ad7431cb8bc1 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:02:21 +0100 Subject: [PATCH 07/21] fix configFile naming --- example_config.yml | 4 ++-- main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example_config.yml b/example_config.yml index 6091157..9804e3e 100644 --- a/example_config.yml +++ b/example_config.yml @@ -4,7 +4,7 @@ metrics: port: 9090 # port for prometheus metrics, served on /metrics and / gateways: - - config-file: "config_holesky.yml" + - configFile: "config_holesky.yml" name: "Holesky gateway" - - config-file: "config_sepolia.yml" + - configFile: "config_sepolia.yml" name: "Sepolia gateway" \ No newline at end of file diff --git a/main.go b/main.go index c3ba353..c010cbf 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ type Config struct { } type GatewayConfig struct { - ConfigFile string `yaml:"config-file"` + ConfigFile string `yaml:"configFile"` Name string `yaml:"name"` } From faab80975732a77e7a709f7bc436400aa288e01f Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:06:16 +0100 Subject: [PATCH 08/21] fix lint --- internal/util/util.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/util/util.go b/internal/util/util.go index 3ecc9be..941e73b 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -47,5 +47,8 @@ func LoadYamlFile[T any](pathOrURL string) (*T, error) { func isValidURL(toTest string) bool { u, err := url.Parse(toTest) - return err == nil && u.Scheme != "" && u.Host != "" + if err != nil { + return false + } + return u.Scheme != "" && u.Host != "" } From c688b1ea8f5083e115ef5f585158ba67cb30ed69 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:10:00 +0100 Subject: [PATCH 09/21] rename go module --- go.mod | 2 +- go.sum | 16 ++++++++++++++++ internal/proxy/nodeprovider.go | 2 +- internal/rpcgateway/config.go | 4 ++-- internal/rpcgateway/rpcgateway.go | 6 +++--- main.go | 8 +++++--- 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index f010d0c..16afa0c 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/0xProject/rpc-gateway +module github.com/sygmaprotocol/rpc-gateway go 1.21 diff --git a/go.sum b/go.sum index 6d52ce5..3450746 100644 --- a/go.sum +++ b/go.sum @@ -3,7 +3,9 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/caitlinelfring/go-env-default v1.1.0 h1:bhDfXmUolvcIGfQCX8qevQX8wxC54NGz0aimoUnhvDM= github.com/caitlinelfring/go-env-default v1.1.0/go.mod h1:tESXPr8zFPP/cRy3cwxrHBmjJIf2A1x/o4C9CET2rEk= github.com/carlmjohnson/deque v0.23.1 h1:X2HOJM9xcglY03deMZ0oZ1V2xtbqYV7dJDnZiSZN4Ak= @@ -13,16 +15,21 @@ github.com/carlmjohnson/flowmatic v0.23.4/go.mod h1:Jpvyl591Dvkt9chYpnVupjxlKvqk github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= +github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.13.13 h1:KYn9w7pEWRI9oyZOzO94OVbctSusPByHdFDPj634jII= github.com/ethereum/go-ethereum v1.13.13/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -35,6 +42,7 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -44,8 +52,11 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -59,6 +70,7 @@ github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -66,6 +78,7 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= @@ -85,6 +98,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= @@ -95,8 +109,10 @@ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/internal/proxy/nodeprovider.go b/internal/proxy/nodeprovider.go index 53c38e9..c95bafd 100644 --- a/internal/proxy/nodeprovider.go +++ b/internal/proxy/nodeprovider.go @@ -5,8 +5,8 @@ import ( "net/http/httputil" "strings" - "github.com/0xProject/rpc-gateway/internal/middleware" "github.com/go-http-utils/headers" + "github.com/sygmaprotocol/rpc-gateway/internal/middleware" ) type NodeProviderConnectionHTTPConfig struct { diff --git a/internal/rpcgateway/config.go b/internal/rpcgateway/config.go index ce8384b..d892743 100644 --- a/internal/rpcgateway/config.go +++ b/internal/rpcgateway/config.go @@ -1,8 +1,8 @@ package rpcgateway import ( - "github.com/0xProject/rpc-gateway/internal/metrics" - "github.com/0xProject/rpc-gateway/internal/proxy" + "github.com/sygmaprotocol/rpc-gateway/internal/metrics" + "github.com/sygmaprotocol/rpc-gateway/internal/proxy" ) type RPCGatewayConfig struct { //nolint:revive diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index df37ee3..14590fc 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -3,19 +3,19 @@ package rpcgateway import ( "context" "fmt" - "github.com/0xProject/rpc-gateway/internal/util" + "github.com/sygmaprotocol/rpc-gateway/internal/util" "log/slog" "net/http" "os" "time" - "github.com/0xProject/rpc-gateway/internal/metrics" - "github.com/0xProject/rpc-gateway/internal/proxy" "github.com/carlmjohnson/flowmatic" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httplog/v2" "github.com/pkg/errors" + "github.com/sygmaprotocol/rpc-gateway/internal/metrics" + "github.com/sygmaprotocol/rpc-gateway/internal/proxy" ) type RPCGateway struct { diff --git a/main.go b/main.go index c010cbf..0439762 100644 --- a/main.go +++ b/main.go @@ -3,15 +3,15 @@ package main import ( "context" "fmt" - "github.com/0xProject/rpc-gateway/internal/metrics" - "github.com/0xProject/rpc-gateway/internal/util" + "github.com/sygmaprotocol/rpc-gateway/internal/metrics" + "github.com/sygmaprotocol/rpc-gateway/internal/util" "os" "os/signal" "sync" "syscall" - "github.com/0xProject/rpc-gateway/internal/rpcgateway" "github.com/pkg/errors" + "github.com/sygmaprotocol/rpc-gateway/internal/rpcgateway" "github.com/urfave/cli/v2" ) @@ -65,6 +65,7 @@ func main() { } wg.Wait() + return nil }, } @@ -86,5 +87,6 @@ func startGateway(ctx context.Context, config GatewayConfig, server *metrics.Ser } <-ctx.Done() + return errors.Wrap(service.Stop(ctx), fmt.Sprintf("cannot stop %s rpc-gateway", config.Name)) } From 0f2db4b34b70a4efcd3784d603ece9edbf04fa80 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:14:28 +0100 Subject: [PATCH 10/21] fix imports --- internal/rpcgateway/rpcgateway.go | 3 ++- internal/util/util.go | 3 ++- main.go | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 14590fc..35da94d 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -3,12 +3,13 @@ package rpcgateway import ( "context" "fmt" - "github.com/sygmaprotocol/rpc-gateway/internal/util" "log/slog" "net/http" "os" "time" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/carlmjohnson/flowmatic" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" diff --git a/internal/util/util.go b/internal/util/util.go index 941e73b..efd7c5e 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -2,11 +2,12 @@ package util import ( "errors" - "gopkg.in/yaml.v2" "io" "net/http" "net/url" "os" + + "gopkg.in/yaml.v2" ) // LoadYamlFile is refactored to use a generic type T. diff --git a/main.go b/main.go index 0439762..708b2d6 100644 --- a/main.go +++ b/main.go @@ -3,13 +3,14 @@ package main import ( "context" "fmt" - "github.com/sygmaprotocol/rpc-gateway/internal/metrics" - "github.com/sygmaprotocol/rpc-gateway/internal/util" "os" "os/signal" "sync" "syscall" + "github.com/sygmaprotocol/rpc-gateway/internal/metrics" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/pkg/errors" "github.com/sygmaprotocol/rpc-gateway/internal/rpcgateway" "github.com/urfave/cli/v2" From 9d79636f5d95edf5977526f7c6c5f8396cda6ef0 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:17:41 +0100 Subject: [PATCH 11/21] fix import lint error: --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index eb88fa9..bf18b7f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,7 +66,7 @@ linters-settings: main: allow: - "$gostd" - - github.com/0xProject/rpc-gateway + - github.com/sygmaprotocol/rpc-gateway - github.com/Shopify/toxiproxy - github.com/ethereum/go-ethereum - github.com/gorilla/mux From 28e72d853b211b5c23016998dfb645e08df465b1 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:20:58 +0100 Subject: [PATCH 12/21] fix http lint error --- internal/util/util.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/util/util.go b/internal/util/util.go index efd7c5e..750fd03 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -17,7 +17,11 @@ func LoadYamlFile[T any](pathOrURL string) (*T, error) { var err error if isValidURL(pathOrURL) { - resp, err := http.Get(pathOrURL) + req, err := http.NewRequest(http.MethodGet, pathOrURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } @@ -51,5 +55,6 @@ func isValidURL(toTest string) bool { if err != nil { return false } + return u.Scheme != "" && u.Host != "" } From e0eb0ba8cd413c6017fcaf09a37c7bfc64c07c12 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:23:51 +0100 Subject: [PATCH 13/21] fix nested lint --- internal/util/util.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/internal/util/util.go b/internal/util/util.go index 750fd03..1c83920 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -17,21 +17,7 @@ func LoadYamlFile[T any](pathOrURL string) (*T, error) { var err error if isValidURL(pathOrURL) { - req, err := http.NewRequest(http.MethodGet, pathOrURL, nil) - if err != nil { - return nil, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, errors.New("failed to fetch config from URL") - } - - data, err = io.ReadAll(resp.Body) + data, err = loadFileFromURL(pathOrURL) if err != nil { return nil, err } @@ -50,6 +36,28 @@ func LoadYamlFile[T any](pathOrURL string) (*T, error) { return &config, nil } +func loadFileFromURL(pathOrURL string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, pathOrURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("failed to fetch config from URL") + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return data, nil +} + func isValidURL(toTest string) bool { u, err := url.Parse(toTest) if err != nil { From 070749fc97c404f74b0c2843f34ee873d5ebac7c Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:25:33 +0100 Subject: [PATCH 14/21] remove deploy action --- .github/workflows/deploy.yaml | 77 ----------------------------------- internal/util/util.go | 1 + 2 files changed, 1 insertion(+), 77 deletions(-) delete mode 100644 .github/workflows/deploy.yaml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index a9b6981..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: deploy - -on: - workflow_dispatch: - inputs: - commit: - description: Commit SHA to deploy. Default to main branch's commit. - environment: - type: choice - options: - - rpc-gateway-arbitrum - - rpc-gateway-avalanche - - rpc-gateway-bsc - - rpc-gateway-celo - - rpc-gateway-ethereum - - rpc-gateway-ethereum-staging - - rpc-gateway-fantom - - rpc-gateway-optimism - - rpc-gateway-polygon - required: true - -env: - IMAGE_NAME: 883408475785.dkr.ecr.us-east-1.amazonaws.com/rpc-gateway - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Validate Input - id: validate_input - uses: actions/github-script@v7 - with: - script: | - const { environment, commit } = context.payload.inputs - core.setOutput('environment', environment) - core.setOutput('commit', commit ? commit : context.sha) - - uses: jwalton/gh-find-current-pr@v1 - id: findPr - with: - state: closed - - name: Checkout 0x-main-infra repo - uses: actions/checkout@v4 - with: - token: ${{ secrets.PAT }} - repository: 0xProject/0x-main-infra - - name: Update GitOps repo - id: update_gitops_repo - run: | - cd 'clusters/main-cluster/kubernetes/${{ steps.validate_input.outputs.environment }}' - # Read current image tag - current_image_tag=$(yq e '.images[] | select(.name | contains("rpc-gateway")) | .newTag' kustomization.yaml) - - # Update image tags - kustomize edit set image ${IMAGE_NAME}:${{ steps.validate_input.outputs.commit }} - - # Output for next steps - echo "##[set-output name=current_image_tag;]${current_image_tag}" - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.PAT }} - author: ${{ github.event.sender.login }} <${{ github.event.sender.login }}@users.noreply.github.com> - commit-message: 'rpc-gateway deploy to ${{ steps.validate_input.outputs.environment }}: ${{ steps.validate_input.outputs.commit }}' - title: '[rpc-gateway][deploy][${{ steps.validate_input.outputs.environment }}] Revision ${{ steps.validate_input.outputs.commit }}' - branch: 'rpc-gateway/deploy/${{ steps.validate_input.outputs.environment }}' - body: | - ### Deployment Details - Triggered by @${{ github.event.sender.login }} - PR: https://github.com/${{ github.repository }}/pull/${{ steps.findPr.outputs.number }} - - **Current version:** https://github.com/${{ github.repository }}/commit/${{ steps.update_gitops_repo.outputs.current_image_tag }} - **Target version:** https://github.com/${{ github.repository }}/commit/${{ steps.validate_input.outputs.commit }} - - Comparison: https://github.com/${{ github.repository }}/compare/${{ steps.update_gitops_repo.outputs.current_image_tag }}...${{ steps.validate_input.outputs.commit }} - - name: Check outputs - run: | - echo "Pull Request Number - ${{ env.PULL_REQUEST_NUMBER }}" diff --git a/internal/util/util.go b/internal/util/util.go index 1c83920..0e7e6af 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -55,6 +55,7 @@ func loadFileFromURL(pathOrURL string) ([]byte, error) { if err != nil { return nil, err } + return data, nil } From 2a020d361dad808bfc9ce670ea822b5a2d3a4f4d Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:26:35 +0100 Subject: [PATCH 15/21] remove docker wokflow --- .github/workflows/docker.yaml | 53 ----------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/workflows/docker.yaml diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml deleted file mode 100644 index d7c0754..0000000 --- a/.github/workflows/docker.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: docker - -on: - pull_request: - branches: - - main - push: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: hadolint/hadolint-action@v3.1.0 - with: - recursive: true - ignore: DL3018,DL3019 - - build: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v4 - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWSROLE }} - aws-region: ${{ secrets.AWSREGION }} - if: ${{ github.actor != 'dependabot[bot]' }} - - uses: aws-actions/amazon-ecr-login@v2 - if: ${{ github.actor != 'dependabot[bot]' }} - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/build-push-action@v5 - with: - push: ${{ github.ref == 'refs/heads/main' }} - platforms: linux/amd64,linux/arm64 - tags: ${{ secrets.REGISTRY }}/rpc-gateway:${{ github.sha }} - if: ${{ github.actor != 'dependabot[bot]' }} - - uses: docker/build-push-action@v5 - with: - push: false - platforms: linux/amd64,linux/arm64 - tags: ${{ secrets.REGISTRY }}/rpc-gateway:${{ github.sha }} - if: ${{ github.actor == 'dependabot[bot]' }} From e08df132643e5462ce5964f7ea3b19d4c8dc5a8d Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 12:45:32 +0100 Subject: [PATCH 16/21] revert vulncheck --- .github/workflows/golang.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index 9c48c2d..ccb8819 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -35,15 +35,15 @@ jobs: RPC_GATEWAY_NODE_URL_1: ${{ secrets.RPC_GATEWAY_NODE_URL_1 }} RPC_GATEWAY_NODE_URL_2: ${{ secrets.RPC_GATEWAY_NODE_URL_2 }} -# vulncheck: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/setup-go@v5 -# with: -# go-version: ${{ env.GOLANGVERSION }} -# - uses: actions/checkout@v4 -# - uses: 0xProject/setup-govulncheck@v2 -# - run: govulncheck ./... + vulncheck: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GOLANGVERSION }} + - uses: actions/checkout@v4 + - uses: 0xProject/setup-govulncheck@v2 + - run: govulncheck ./... lint: runs-on: ubuntu-latest From bab84139413d68e331cdd926cdebb13e2a68f3cd Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 13:37:27 +0100 Subject: [PATCH 17/21] refactor for each network to be on different handle --- internal/proxy/config.go | 2 +- internal/rpcgateway/rpcgateway.go | 74 ++++++------------------------- main.go | 42 ++++++++++++++++-- 3 files changed, 54 insertions(+), 64 deletions(-) diff --git a/internal/proxy/config.go b/internal/proxy/config.go index 2fd6f91..b376100 100644 --- a/internal/proxy/config.go +++ b/internal/proxy/config.go @@ -12,7 +12,7 @@ type HealthCheckConfig struct { } type ProxyConfig struct { // nolint:revive - Port string `yaml:"port"` + Path string `yaml:"path"` UpstreamTimeout time.Duration `yaml:"upstreamTimeout"` } diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 35da94d..e498549 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -3,32 +3,21 @@ package rpcgateway import ( "context" "fmt" + "github.com/sygmaprotocol/rpc-gateway/internal/util" "log/slog" - "net/http" "os" - "time" - - "github.com/sygmaprotocol/rpc-gateway/internal/util" "github.com/carlmjohnson/flowmatic" "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" - "github.com/go-chi/httplog/v2" "github.com/pkg/errors" "github.com/sygmaprotocol/rpc-gateway/internal/metrics" "github.com/sygmaprotocol/rpc-gateway/internal/proxy" ) type RPCGateway struct { - config RPCGatewayConfig - proxy *proxy.Proxy - hcm *proxy.HealthCheckManager - server *http.Server - metrics *metrics.Server -} - -func (r *RPCGateway) ServeHTTP(w http.ResponseWriter, req *http.Request) { - r.server.Handler.ServeHTTP(w, req) + config RPCGatewayConfig + proxy *proxy.Proxy + hcm *proxy.HealthCheckManager } func (r *RPCGateway) Start(c context.Context) error { @@ -36,12 +25,6 @@ func (r *RPCGateway) Start(c context.Context) error { func() error { return errors.Wrap(r.hcm.Start(c), "failed to start health check manager") }, - func() error { - return errors.Wrap(r.server.ListenAndServe(), "failed to start rpc-gateway") - }, - func() error { - return errors.Wrap(r.metrics.Start(), "failed to start metrics server") - }, ) } @@ -50,27 +33,15 @@ func (r *RPCGateway) Stop(c context.Context) error { func() error { return errors.Wrap(r.hcm.Stop(c), "failed to stop health check manager") }, - func() error { - return errors.Wrap(r.server.Close(), "failed to stop rpc-gateway") - }, - func() error { - return errors.Wrap(r.metrics.Stop(), "failed to stop metrics server") - }, ) } -func NewRPCGateway(config RPCGatewayConfig, metricsServer *metrics.Server) (*RPCGateway, error) { +func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux, metricsServer *metrics.Server) (*RPCGateway, error) { logLevel := slog.LevelWarn if os.Getenv("DEBUG") == "true" { logLevel = slog.LevelDebug } - logger := httplog.NewLogger("rpc-gateway", httplog.Options{ - JSON: true, - RequestHeaders: true, - LogLevel: logLevel, - }) - hcm, err := proxy.NewHealthCheckManager( proxy.HealthCheckManagerConfig{ Targets: config.Targets, @@ -97,42 +68,25 @@ func NewRPCGateway(config RPCGatewayConfig, metricsServer *metrics.Server) (*RPC return nil, errors.Wrap(err, "proxy failed") } - r := chi.NewRouter() - r.Use(httplog.RequestLogger(logger)) - - // Recoverer is a middleware that recovers from panics, logs the panic (and - // a backtrace), and returns a HTTP 500 (Internal Server Error) status if - // possible. Recoverer prints a request ID if one is provided. - // - r.Use(middleware.Recoverer) - - r.Handle("/", proxy) + router.Handle(fmt.Sprintf("/%s", config.Proxy.Path), proxy) return &RPCGateway{ - config: config, - proxy: proxy, - hcm: hcm, - metrics: metricsServer, - server: &http.Server{ - Addr: fmt.Sprintf(":%s", config.Proxy.Port), - Handler: r, - WriteTimeout: time.Second * 15, - ReadTimeout: time.Second * 15, - ReadHeaderTimeout: time.Second * 5, - }, + config: config, + proxy: proxy, + hcm: hcm, }, nil } // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. -func NewRPCGatewayFromConfigFile(s string, server *metrics.Server) (*RPCGateway, error) { - config, err := util.LoadYamlFile[RPCGatewayConfig](s) +func NewRPCGatewayFromConfigFile(fileOrUrl string, router *chi.Mux, metricsServer *metrics.Server) (*RPCGateway, error) { + config, err := util.LoadYamlFile[RPCGatewayConfig](fileOrUrl) if err != nil { return nil, errors.Wrap(err, "failed to load config") } - fmt.Println("Starting RPC Gateway for " + config.Name + " on port: " + config.Proxy.Port) + fmt.Println("Starting RPC Gateway for " + config.Name + " on path: /" + config.Proxy.Path) - // Pass the metrics server as an argument to NewRPCGateway. - return NewRPCGateway(*config, server) + // Pass the metrics router as an argument to NewRPCGateway. + return NewRPCGateway(*config, router, metricsServer) } diff --git a/main.go b/main.go index 708b2d6..ebb7dbd 100644 --- a/main.go +++ b/main.go @@ -3,10 +3,16 @@ package main import ( "context" "fmt" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/httplog/v2" + "log/slog" + "net/http" "os" "os/signal" "sync" "syscall" + "time" "github.com/sygmaprotocol/rpc-gateway/internal/metrics" "github.com/sygmaprotocol/rpc-gateway/internal/util" @@ -22,6 +28,7 @@ type MetricsConfig struct { type Config struct { Metrics MetricsConfig `yaml:"metrics"` + Port string `yaml:"port"` Gateways []GatewayConfig `yaml:"gateways"` } @@ -51,20 +58,49 @@ func main() { return errors.Wrap(err, "failed to load config") } + logLevel := slog.LevelWarn + if os.Getenv("DEBUG") == "true" { + logLevel = slog.LevelDebug + } + + logger := httplog.NewLogger("rpc-gateway", httplog.Options{ + JSON: true, + RequestHeaders: true, + LogLevel: logLevel, + }) + metricsServer := metrics.NewServer(metrics.Config{Port: uint(config.Metrics.Port)}) + r := chi.NewRouter() + r.Use(httplog.RequestLogger(logger)) + r.Use(middleware.Recoverer) + server := &http.Server{ + Addr: fmt.Sprintf(":%s", config.Port), + Handler: r, + WriteTimeout: time.Second * 15, + ReadTimeout: time.Second * 15, + ReadHeaderTimeout: time.Second * 5, + } + defer server.Close() + var wg sync.WaitGroup for _, gatewayConfig := range config.Gateways { wg.Add(1) go func(gwConfig GatewayConfig) { defer wg.Done() - err := startGateway(c, gwConfig, metricsServer) + err := startGateway(c, gwConfig, r, metricsServer) if err != nil { fmt.Fprintf(os.Stderr, "error starting gateway '%s': %v\n", gwConfig.Name, err) } }(gatewayConfig) } + fmt.Println("Starting RPC Gateway server on port: " + config.Port) + err = server.ListenAndServe() + if err != nil { + return err + } + wg.Wait() return nil @@ -76,8 +112,8 @@ func main() { } } -func startGateway(ctx context.Context, config GatewayConfig, server *metrics.Server) error { - service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, server) +func startGateway(ctx context.Context, config GatewayConfig, router *chi.Mux, metricsServer *metrics.Server) error { + service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, router, metricsServer) if err != nil { return errors.Wrap(err, fmt.Sprintf("%s rpc-gateway failed", config.Name)) } From ac07bc27e19826f029732b2a1cc4cc850a8540eb Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 13:45:47 +0100 Subject: [PATCH 18/21] fix imports --- internal/rpcgateway/rpcgateway.go | 3 ++- main.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index e498549..4cae80b 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -3,10 +3,11 @@ package rpcgateway import ( "context" "fmt" - "github.com/sygmaprotocol/rpc-gateway/internal/util" "log/slog" "os" + "github.com/sygmaprotocol/rpc-gateway/internal/util" + "github.com/carlmjohnson/flowmatic" "github.com/go-chi/chi/v5" "github.com/pkg/errors" diff --git a/main.go b/main.go index ebb7dbd..7f71050 100644 --- a/main.go +++ b/main.go @@ -3,9 +3,6 @@ package main import ( "context" "fmt" - "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" - "github.com/go-chi/httplog/v2" "log/slog" "net/http" "os" @@ -14,6 +11,10 @@ import ( "syscall" "time" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/httplog/v2" + "github.com/sygmaprotocol/rpc-gateway/internal/metrics" "github.com/sygmaprotocol/rpc-gateway/internal/util" From 71caed29c55b1bd65e5cf0c453c9f2292d0647cd Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 13:59:44 +0100 Subject: [PATCH 19/21] fix bug with metrics server not starting --- internal/rpcgateway/rpcgateway.go | 7 +++---- main.go | 13 ++++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 4cae80b..37ecb8e 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -11,7 +11,6 @@ import ( "github.com/carlmjohnson/flowmatic" "github.com/go-chi/chi/v5" "github.com/pkg/errors" - "github.com/sygmaprotocol/rpc-gateway/internal/metrics" "github.com/sygmaprotocol/rpc-gateway/internal/proxy" ) @@ -37,7 +36,7 @@ func (r *RPCGateway) Stop(c context.Context) error { ) } -func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux, metricsServer *metrics.Server) (*RPCGateway, error) { +func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux) (*RPCGateway, error) { logLevel := slog.LevelWarn if os.Getenv("DEBUG") == "true" { logLevel = slog.LevelDebug @@ -80,7 +79,7 @@ func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux, metricsServer *metr // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. -func NewRPCGatewayFromConfigFile(fileOrUrl string, router *chi.Mux, metricsServer *metrics.Server) (*RPCGateway, error) { +func NewRPCGatewayFromConfigFile(fileOrUrl string, router *chi.Mux) (*RPCGateway, error) { config, err := util.LoadYamlFile[RPCGatewayConfig](fileOrUrl) if err != nil { return nil, errors.Wrap(err, "failed to load config") @@ -89,5 +88,5 @@ func NewRPCGatewayFromConfigFile(fileOrUrl string, router *chi.Mux, metricsServe fmt.Println("Starting RPC Gateway for " + config.Name + " on path: /" + config.Proxy.Path) // Pass the metrics router as an argument to NewRPCGateway. - return NewRPCGateway(*config, router, metricsServer) + return NewRPCGateway(*config, router) } diff --git a/main.go b/main.go index 7f71050..2147e1c 100644 --- a/main.go +++ b/main.go @@ -71,6 +71,13 @@ func main() { }) metricsServer := metrics.NewServer(metrics.Config{Port: uint(config.Metrics.Port)}) + go func() { + err = metricsServer.Start() + defer metricsServer.Stop() + if err != nil { + fmt.Fprintf(os.Stderr, "error starting metrics server: %v\n", err) + } + }() r := chi.NewRouter() r.Use(httplog.RequestLogger(logger)) @@ -89,7 +96,7 @@ func main() { wg.Add(1) go func(gwConfig GatewayConfig) { defer wg.Done() - err := startGateway(c, gwConfig, r, metricsServer) + err := startGateway(c, gwConfig, r) if err != nil { fmt.Fprintf(os.Stderr, "error starting gateway '%s': %v\n", gwConfig.Name, err) } @@ -113,8 +120,8 @@ func main() { } } -func startGateway(ctx context.Context, config GatewayConfig, router *chi.Mux, metricsServer *metrics.Server) error { - service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, router, metricsServer) +func startGateway(ctx context.Context, config GatewayConfig, router *chi.Mux) error { + service, err := rpcgateway.NewRPCGatewayFromConfigFile(config.ConfigFile, router) if err != nil { return errors.Wrap(err, fmt.Sprintf("%s rpc-gateway failed", config.Name)) } From 25ca19e70150b80936d857c83a2a802422445144 Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 14:01:32 +0100 Subject: [PATCH 20/21] fix linter --- internal/rpcgateway/rpcgateway.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rpcgateway/rpcgateway.go b/internal/rpcgateway/rpcgateway.go index 37ecb8e..29ae0bd 100644 --- a/internal/rpcgateway/rpcgateway.go +++ b/internal/rpcgateway/rpcgateway.go @@ -79,8 +79,8 @@ func NewRPCGateway(config RPCGatewayConfig, router *chi.Mux) (*RPCGateway, error // NewRPCGatewayFromConfigFile creates an instance of RPCGateway from provided // configuration file. -func NewRPCGatewayFromConfigFile(fileOrUrl string, router *chi.Mux) (*RPCGateway, error) { - config, err := util.LoadYamlFile[RPCGatewayConfig](fileOrUrl) +func NewRPCGatewayFromConfigFile(fileOrURL string, router *chi.Mux) (*RPCGateway, error) { + config, err := util.LoadYamlFile[RPCGatewayConfig](fileOrURL) if err != nil { return nil, errors.Wrap(err, "failed to load config") } From 8495fac13850eb86714f996ab87021ce9fb3c1be Mon Sep 17 00:00:00 2001 From: MakMuftic Date: Wed, 13 Mar 2024 14:03:42 +0100 Subject: [PATCH 21/21] handle error lint fix --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 2147e1c..9c5e369 100644 --- a/main.go +++ b/main.go @@ -73,7 +73,12 @@ func main() { metricsServer := metrics.NewServer(metrics.Config{Port: uint(config.Metrics.Port)}) go func() { err = metricsServer.Start() - defer metricsServer.Stop() + defer func(metricsServer *metrics.Server) { + err := metricsServer.Stop() + if err != nil { + fmt.Fprintf(os.Stderr, "error stopping metrics server: %v\n", err) + } + }(metricsServer) if err != nil { fmt.Fprintf(os.Stderr, "error starting metrics server: %v\n", err) }