From f678fad573b60f340e6d5c71ac41d35d706ad512 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Mon, 7 Oct 2024 19:12:00 +0200 Subject: [PATCH 1/6] wip --- .gitignore | 2 +- Dockerfile | 28 ++ adapters/database/service.go | 384 +++++++++++++++++++++++ adapters/database/service_test.go | 73 +++++ adapters/database/types.go | 110 +++++++ adapters/secrets/service.go | 44 +++ application/config_test.go | 188 +++++++++++ application/config_utils.go | 35 +++ application/service.go | 54 ++++ cmd/httpserver/main.go | 21 +- database/database.go | 54 ---- database/database_test.go | 50 --- database/migrations/001_init_database.go | 21 -- database/migrations/migration.go | 12 - database/types.go | 29 -- database/types_test.go | 19 -- database/vars/tables.go | 11 - domain/measurements-jq.json | 1 + domain/measurements.json | 1 + domain/types.go | 55 ++++ domain/types_test.go | 29 ++ go.mod | 6 + go.sum | 199 ++++++++++++ httpserver.dockerfile | 23 -- httpserver/server.go | 20 +- ports/http_server.go | 150 +++++++++ ports/types.go | 77 +++++ ports/types_test.go | 27 ++ schema/000_init.sql | 97 ++++++ 29 files changed, 1590 insertions(+), 230 deletions(-) create mode 100644 Dockerfile create mode 100644 adapters/database/service.go create mode 100644 adapters/database/service_test.go create mode 100644 adapters/database/types.go create mode 100644 adapters/secrets/service.go create mode 100644 application/config_test.go create mode 100644 application/config_utils.go create mode 100644 application/service.go delete mode 100644 database/database.go delete mode 100644 database/database_test.go delete mode 100644 database/migrations/001_init_database.go delete mode 100644 database/migrations/migration.go delete mode 100644 database/types.go delete mode 100644 database/types_test.go delete mode 100644 database/vars/tables.go create mode 100644 domain/measurements-jq.json create mode 100644 domain/measurements.json create mode 100644 domain/types.go create mode 100644 domain/types_test.go delete mode 100644 httpserver.dockerfile create mode 100644 ports/http_server.go create mode 100644 ports/types.go create mode 100644 ports/types_test.go create mode 100644 schema/000_init.sql diff --git a/.gitignore b/.gitignore index 30e763c..1b1fcf3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ # IDE .vscode - +.idea # Builds /build .aider* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a231157 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.23-alpine AS builder +ARG VERSION +RUN apk add --no-cache gcc sqlite-dev musl-dev +WORKDIR /build +# First only add go.mod and go.sum, then run go mod download to cache dependencies +# in a separate layer. +ADD go.mod go.sum /build/ +RUN --mount=type=cache,target=/root/.cache/go-build go mod download +# Now add the rest of the source code and build the application. +ADD . /build/ +RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=1 GOOS=linux \ + go build \ + -trimpath \ + -ldflags "-s -X github.com/flashbots/builder-hub/common.Version=${VERSION} -w -extldflags \"-static\"" \ + -v \ + -o builder-hub \ + cmd/httpserver/main.go + +FROM alpine:latest +RUN apk update && apk upgrade +RUN apk add --no-cache sqlite-dev +# See http://stackoverflow.com/questions/34729748/installed-go-binary-not-found-in-path-on-alpine-linux-docker +RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +WORKDIR /app +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /build/builder-hub /app/builder-hub +CMD ["/app/builder-hub"] diff --git a/adapters/database/service.go b/adapters/database/service.go new file mode 100644 index 0000000..30f28c7 --- /dev/null +++ b/adapters/database/service.go @@ -0,0 +1,384 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "net" + + "github.com/flashbots/builder-hub/domain" + "github.com/jackc/pgtype" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" +) + +type Service struct { + DB *sqlx.DB +} + +func NewDatabaseService(dsn string) (*Service, error) { + db, err := sqlx.Connect("postgres", dsn) + if err != nil { + return nil, err + } + + db.DB.SetMaxOpenConns(50) + db.DB.SetMaxIdleConns(10) + db.DB.SetConnMaxIdleTime(0) + + dbService := &Service{DB: db} //nolint:exhaustruct + return dbService, err +} + +func (s *Service) Close() error { + return s.DB.Close() +} + +// +//// GetAllowedMeasurements retrieves all active measurements +//func (s *Service) GetAllowedMeasurements() ([]Measurement, error) { +// var measurements []Measurement +// err := s.DB.Select(&measurements, ` +// SELECT id, measurement +// FROM measurements_whitelist +// WHERE is_active = true +// `) +// return measurements, err +//} +// +//// GetActiveBuilders retrieves all active builders with their IPs and public keys +//func (s *Service) GetActiveBuilders() ([]ActiveBuilder, error) { +// var builders []ActiveBuilder +// err := s.DB.Select(&builders, ` +// SELECT b.id AS builder_id, b.name, iw.ip_address, scr.tls_pubkey, scr.ecdsa_pubkey +// FROM builders b +// JOIN ip_whitelist iw ON b.id = iw.builder_id +// JOIN service_credential_registrations scr ON iw.id = scr.ip_whitelist_id +// WHERE iw.is_active = true AND scr.service = 'builder' +// `) +// return builders, err +//} +// +//// GetConfigByIP retrieves the config for a builder based on their IP address +//func (s *Service) GetConfigByIP(ip string) (BuilderConfig, error) { +// var config BuilderConfig +// err := s.DB.Get(&config, ` +// SELECT bc.builder_id, bc.config +// FROM builder_configs bc +// JOIN ip_whitelist iw ON bc.builder_id = iw.builder_id +// WHERE iw.ip_address = $1 AND iw.is_active = true AND bc.is_active = true +// `, ip) +// return config, err +//} +// +//// RegisterNewCredentials registers new credentials for a service +//func (s *Service) RegisterNewCredentials(ip, service string, tlsPubKey, ecdsaPubKey []byte) error { +// _, err := s.DB.Exec(` +// INSERT INTO service_credential_registrations (ip_whitelist_id, service, tls_pubkey, ecdsa_pubkey) +// SELECT id, $2, $3, $4 +// FROM ip_whitelist +// WHERE ip_address = $1 AND is_active = true +// `, ip, service, tlsPubKey, ecdsaPubKey) +// return err +//} +// +//// GetBuilderByIP retrieves an active builder by their IP address +//func (s *Service) GetActiveBuilderByIP(ip net.IP) (*ActiveBuilder, error) { +// var paramIP pgtype.Inet +// err := paramIP.Set(ip) +// if err != nil { +// return nil, err +// } +// +// var builder ActiveBuilder +// err = s.DB.Get(&builder, ` +// SELECT b.id AS builder_id, b.name, iw.ip_address, scr.tls_pubkey, scr.ecdsa_pubkey +// FROM builders b +// JOIN ip_whitelist iw ON b.id = iw.builder_id +// JOIN service_credential_registrations scr ON iw.id = scr.ip_whitelist_id +// WHERE iw.ip_address = $1 AND iw.is_active = true AND scr.service = 'builder' +// `, paramIP) +// +// if err != nil { +// return nil, fmt.Errorf("error getting builder by IP: %w", err) +// } +// +// return &builder, nil +//} +// +//type IPWhitelistEntry struct { +// BuilderID int `db:"builder_id"` +// IPAddress pgtype.Inet `db:"ip_address"` +// IsActive bool `db:"is_active"` +// ValidFrom time.Time `db:"valid_from"` +// ValidTo *time.Time `db:"valid_to"` +//} +// +//// use for validation if needed +//func (s *Service) GetIPWhitelistByIP(ip net.IP) (*IPWhitelistEntry, error) { +// +// var paramIP pgtype.Inet +// err := paramIP.Set(ip) +// if err != nil { +// return nil, err +// } +// +// var entry IPWhitelistEntry +// err = s.DB.Get(&entry, ` +// SELECT builder_id, ip_address, is_active, valid_from, valid_to +// FROM ip_whitelist +// WHERE ip_address = $1 +// `, paramIP) +// +// if err != nil { +// return nil, fmt.Errorf("error getting IP whitelist entry: %w", err) +// } +// +// //entry.IPAddress.IPNet.IP +// return &entry, nil +//} + +// GetMeasurementByOIDAndHash retrieves a measurement by OID and hash +func (s *Service) GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) { + var m Measurement + err := s.DB.Get(&m, ` + SELECT * FROM measurements_whitelist + WHERE attestation_type = $1 AND hash = $2 AND is_active = true + `, attestationType, hash) + if errors.Is(err, sql.ErrNoRows) { + return nil, domain.ErrNotFound + } + return convertMeasurementToDomain(m) +} + +// GetBuilderByIP retrieves a builder by IP address +func (s *Service) GetBuilderByIP(ip net.IP) (*domain.Builder, error) { + var paramIP pgtype.Inet + err := paramIP.Set(ip) + if err != nil { + return nil, err + } + + var b Builder + err = s.DB.Get(&b, ` + SELECT * FROM builders + WHERE ip_address = $1 and is_active = true + `, paramIP) + if errors.Is(err, sql.ErrNoRows) { + return nil, domain.ErrNotFound + } + if err != nil { + return nil, err + } + return convertBuilderToDomain(b) +} + +// GetActiveMeasurements retrieves all measurements +func (s *Service) GetActiveMeasurements(ctx context.Context) ([]domain.Measurement, error) { + var measurements []Measurement + err := s.DB.SelectContext(ctx, &measurements, `SELECT * FROM measurements_whitelist WHERE is_active=true`) + var domainMeasurements []domain.Measurement + for _, m := range measurements { + domainM, err := convertMeasurementToDomain(m) + if err != nil { + return nil, err + } + domainMeasurements = append(domainMeasurements, *domainM) + } + return domainMeasurements, err +} + +// GetActiveBuilders retrieves all active builders +func (s *Service) GetActiveBuilders() ([]Builder, error) { + var builders []Builder + err := s.DB.Select(&builders, ` + SELECT * FROM builders + WHERE is_active = true AND deprecated_at IS NULL + `) + return builders, err +} + +// RegisterCredentialsForBuilder registers new credentials for a builder, deprecating all previous credentials +func (s *Service) RegisterCredentialsForBuilder(builderName, service, tlsCert string, ecdsaPubKey []byte) error { + // Start a transaction + tx, err := s.DB.Begin() + if err != nil { + return err + } + defer tx.Rollback() // Rollback the transaction if it's not committed + + // Deprecate all previous credentials for this builder and service + _, err = tx.Exec(` + UPDATE service_credential_registrations + SET is_active = false + WHERE builder_name = $1 AND service = $2 + `, builderName, service) + if err != nil { + return err + } + + // Insert new credentials + var nullableTLSCert sql.NullString + if tlsCert != "" { + nullableTLSCert = sql.NullString{String: tlsCert, Valid: true} + } + + _, err = tx.Exec(` + INSERT INTO service_credential_registrations + (builder_name, service, tls_cert, ecdsa_pubkey, is_active) + VALUES ($1, $2, $3, $4, true) + `, builderName, service, nullableTLSCert, ecdsaPubKey) + if err != nil { + return err + } + + // Commit the transaction + if err = tx.Commit(); err != nil { + return err + } + + return nil +} + +// GetActiveConfigForBuilder retrieves the active config for a builder by name +func (s *Service) GetActiveConfigForBuilder(builderName string) (*BuilderConfig, error) { + var config BuilderConfig + err := s.DB.Get(&config, ` + SELECT * FROM builder_configs + WHERE builder_name = $1 AND is_active = true + `, builderName) + return &config, err +} + +func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ([]domain.BuilderWithServices, error) { + rows, err := s.DB.Query(` + SELECT + b.name, + b.ip_address, + scr.service, + scr.tls_cert, + scr.ecdsa_pubkey + FROM + builders b + LEFT JOIN + service_credential_registrations scr ON b.name = scr.builder_name + WHERE + b.is_active = true AND (scr.is_active = true OR scr.is_active IS NULL) + ORDER BY + b.name, scr.service + `) + if err != nil { + return nil, err + } + defer rows.Close() + + buildersMap := make(map[string]*BuilderWithCredentials) + + for rows.Next() { + var ipAddress pgtype.Inet + var builderName, service string + var tlsCert sql.NullString + var ecdsaPubKey []byte + + err := rows.Scan(&builderName, &ipAddress, &service, &tlsCert, &ecdsaPubKey) + if err != nil { + return nil, err + } + + builder, exists := buildersMap[builderName] + if !exists { + builder = &BuilderWithCredentials{ + Name: builderName, + IPAddress: ipAddress, + } + buildersMap[builderName] = builder + } + + if service != "" { + builder.Credentials = append(builder.Credentials, ServiceCredential{ + Service: service, + TLSCert: tlsCert, + ECDSAPubKey: ecdsaPubKey, + }) + } + } + + if err = rows.Err(); err != nil { + return nil, err + } + + // Convert map to slice + builders := make([]domain.BuilderWithServices, 0, len(buildersMap)) + for _, builder := range buildersMap { + dBuilder, err := toDomainBuilderWithCredentials(*builder) + if err != nil { + return nil, err + } + builders = append(builders, *dBuilder) + } + + return builders, nil +} + +// +//// GetActiveBuildersWithCredentials retrieves all active builders along with their active service credentials +//func (s *Service) GetActiveBuildersServices(ctx context.Context) ([]domain.BuilderWithServices, error) { +// rows, err := s.DB.QueryContext(ctx, ` +// SELECT +// b.name, +// b.ip_address, +// json_agg( +// json_build_object( +// 'service', scr.service, +// 'tls_cert', scr.tls_cert, +// 'ecdsa_pubkey', encode(scr.ecdsa_pubkey, 'base64') +// ) +// ) AS credentials +// FROM +// builders b +// LEFT JOIN +// service_credential_registrations scr ON b.name = scr.builder_name +// WHERE +// b.is_active = true AND scr.is_active = true +// GROUP BY +// b.name, b.ip_address +// `) +// if err != nil { +// return nil, err +// } +// defer rows.Close() +// +// var builders []BuilderWithCredentials +// for rows.Next() { +// var builder BuilderWithCredentials +// var credentialsJSON []byte +// err := rows.Scan(&builder.Name, &builder.IPAddress, &credentialsJSON) +// if err != nil { +// return nil, err +// } +// +// if credentialsJSON != nil { +// err = json.Unmarshal(credentialsJSON, &builder.Credentials) +// if err != nil { +// return nil, err +// } +// } +// +// builders = append(builders, builder) +// } +// +// if err = rows.Err(); err != nil { +// return nil, err +// } +// +// var dBuilders []domain.BuilderWithServices +// for _, b := range builders { +// dBuilder, err := toDomainBuilderWithCredentials(b) +// if err != nil { +// return nil, err +// } +// dBuilders = append(dBuilders, *dBuilder) +// } +// +// return dBuilders, nil +//} diff --git a/adapters/database/service_test.go b/adapters/database/service_test.go new file mode 100644 index 0000000..4df95f0 --- /dev/null +++ b/adapters/database/service_test.go @@ -0,0 +1,73 @@ +package database + +import ( + "context" + "fmt" + "net" + "testing" +) + +func TestGetBuilder(t *testing.T) { + serv, err := NewDatabaseService("postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable") + if err != nil { + t.Errorf("NewDatabaseService() = %v; want nil", err) + } + //t.Run("GetBuilder", func(t *testing.T) { + // _, err = db.Exec("create temporary table t (id serial primary key, ip inet not null);") + // if err != nil { + // log.Fatal(err) + // } + // + // var paramIP pgtype.Inet + // err = paramIP.Set("10.0.0.0/16") + // if err != nil { + // log.Fatal(err) + // } + // + // _, err = db.Exec("insert into t (ip) values ($1);", paramIP) + // if err != nil { + // log.Fatal(err) + // } + // + // var resultIP pgtype.Inet + // err = db.QueryRow("select ip from t").Scan(&resultIP) + // if err != nil { + // log.Fatal(err) + // } + // + // fmt.Println(resultIP.IPNet) + //}) + //t.Run("GetBuilder", func(t *testing.T) { + // t.Run("should return a builder", func(t *testing.T) { + // builder, err := serv.GetBuilderByIP(net.ParseIP("192.168.1.100")) + // if err != nil { + // t.Errorf("GetBuilder() = %v; want nil", err) + // } + // fmt.Println(builder) + // }) + //}) + t.Run("GetBuilder2", func(t *testing.T) { + t.Run("should return a builder", func(t *testing.T) { + whitelist, err := serv.GetBuilderByIP(net.ParseIP("192.168.1.1")) + if err != nil { + t.Errorf("GetIPWhitelistByIP() = %v; want nil", err) + } + fmt.Println(whitelist) + }) + t.Run("get all active builders", func(t *testing.T) { + whitelist, err := serv.GetActiveBuildersWithServiceCredentials(context.Background()) + if err != nil { + t.Errorf("GetIPWhitelistByIP() = %v; want nil", err) + } + fmt.Println(whitelist) + }) + t.Run("get all active measurements", func(t *testing.T) { + whitelist, err := serv.GetActiveMeasurements(context.Background()) + if err != nil { + t.Errorf("GetIPWhitelistByIP() = %v; want nil", err) + } + fmt.Println(whitelist) + }) + }) + +} diff --git a/adapters/database/types.go b/adapters/database/types.go new file mode 100644 index 0000000..66cd1b2 --- /dev/null +++ b/adapters/database/types.go @@ -0,0 +1,110 @@ +package database + +import ( + "database/sql" + "encoding/json" + "time" + + "github.com/flashbots/builder-hub/domain" + "github.com/jackc/pgtype" +) + +type ActiveBuilder struct { + BuilderID int `db:"builder_id"` + Name string `db:"name"` + IPAddress pgtype.Inet `db:"ip_address"` + TLSPubKey []byte `db:"tls_pubkey"` + ECDSAPubKey []byte `db:"ecdsa_pubkey"` +} + +type Measurement struct { + Hash []byte `db:"hash"` + AttestationType string `db:"attestation_type"` + Measurement json.RawMessage `db:"measurement"` + IsActive bool `db:"is_active"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeprecatedAt *time.Time `db:"deprecated_at"` +} + +func convertMeasurementToDomain(measurement Measurement) (*domain.Measurement, error) { + var m domain.Measurement + m.AttestationType = measurement.AttestationType + m.Measurement = make(map[string]string) + err := json.Unmarshal(measurement.Measurement, &m.Measurement) + if err != nil { + return nil, err + } + m.Hash = measurement.Hash + return &m, nil +} + +type Builder struct { + Name string `db:"name"` + IPAddress pgtype.Inet `db:"ip_address"` + IsActive bool `db:"is_active"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeprecatedAt *time.Time `db:"deprecated_at"` +} + +func convertBuilderToDomain(builder Builder) (*domain.Builder, error) { + if builder.IPAddress.IPNet == nil { + return nil, domain.ErrIncorrectBuilder + } + return &domain.Builder{ + Name: builder.Name, + IPAddress: builder.IPAddress.IPNet.IP, + IsActive: builder.IsActive, + }, nil +} + +type ServiceCredentialRegistration struct { + ID int `db:"id"` + BuilderName string `db:"builder_name"` + Service string `db:"service"` + TLSCert string `db:"tls_cert"` + ECDSAPubKey []byte `db:"ecdsa_pubkey"` + CreatedAt time.Time `db:"created_at"` +} + +type BuilderConfig struct { + ID int `db:"id"` + BuilderName string `db:"builder_name"` + Config json.RawMessage `db:"config"` + IsActive bool `db:"is_active"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +type BuilderWithCredentials struct { + Name string + IPAddress pgtype.Inet + Credentials []ServiceCredential +} +type ServiceCredential struct { + Service string + TLSCert sql.NullString + ECDSAPubKey []byte +} + +func toDomainBuilderWithCredentials(builder BuilderWithCredentials) (*domain.BuilderWithServices, error) { + if builder.IPAddress.IPNet == nil { + return nil, domain.ErrIncorrectBuilder + } + s := domain.BuilderWithServices{ + Builder: domain.Builder{ + Name: builder.Name, + IPAddress: builder.IPAddress.IPNet.IP, + IsActive: true, + }, + Services: make([]domain.BuilderServices, 0, len(builder.Credentials)), + } + for _, cred := range builder.Credentials { + s.Services = append(s.Services, domain.BuilderServices{ + TLSCert: cred.TLSCert.String, + ECDSAPubKey: cred.ECDSAPubKey, + }) + } + return &s, nil +} diff --git a/adapters/secrets/service.go b/adapters/secrets/service.go new file mode 100644 index 0000000..db1c66b --- /dev/null +++ b/adapters/secrets/service.go @@ -0,0 +1,44 @@ +package secrets + +import ( + "encoding/json" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/secretsmanager" +) + +type Service struct { + sm *secretsmanager.SecretsManager +} + +func NewService() (*Service, error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-2"), + }) + if err != nil { + return nil, err + } + + // Create a Secrets Manager client + svc := secretsmanager.New(sess) + + return &Service{sm: svc}, nil +} + +func (s *Service) GetSecretValues(secretName string) (map[string]string, error) { + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretName), + } + + result, err := s.sm.GetSecretValue(input) + if err != nil { + return nil, err + } + secretData := make(map[string]string) + err = json.Unmarshal([]byte(*result.SecretString), &secretData) + if err != nil { + return nil, err + } + return secretData, nil +} diff --git a/application/config_test.go b/application/config_test.go new file mode 100644 index 0000000..0bb1295 --- /dev/null +++ b/application/config_test.go @@ -0,0 +1,188 @@ +package application + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +type ExampleConfig struct { + RethChainDownloader struct { + AwsAccessKeyId string `json:"aws_access_key_id"` + AwsSecretAccessKey string `json:"aws_secret_access_key"` + RcloneS3Endpoint string `json:"rclone_s3_endpoint"` + } `json:"reth_chain_downloader"` + OrderflowProxy struct { + FlashbotsOfSigningKey string `json:"flashbots_of_signing_key"` + BuilderPublicIp string `json:"builder_public_ip"` + TlsHosts []string `json:"tls_hosts"` + } `json:"orderflow_proxy"` + Rbuilder struct { + ExtraData string `json:"extra_data"` + RelaySecretKey string `json:"relay_secret_key"` + OptimisticRelaySecretKey string `json:"optimistic_relay_secret_key"` + CoinbaseSecretKey string `json:"coinbase_secret_key"` + AlwaysSeal bool `json:"always_seal"` + Relays []struct { + Name string `json:"name"` + Url string `json:"url"` + UseSszForSubmit bool `json:"use_ssz_for_submit"` + UseGzipForSubmit bool `json:"use_gzip_for_submit"` + Priority int `json:"priority"` + Optimistic bool `json:"optimistic"` + } `json:"relays"` + } `json:"rbuilder"` + Prometheus struct { + ScrapeInterval string `json:"scrape_interval"` + StaticConfigsDefaultLabels []struct { + LabelKey string `json:"label_key"` + LabelValue string `json:"label_value"` + } `json:"static_configs_default_labels"` + LighthouseMetrics struct { + Enabled bool `json:"enabled"` + Targets []string `json:"targets"` + } `json:"lighthouse_metrics"` + RethMetrics struct { + Enabled bool `json:"enabled"` + Targets []string `json:"targets"` + } `json:"reth_metrics"` + RbuilderMetrics struct { + Enabled bool `json:"enabled"` + Targets []string `json:"targets"` + } `json:"rbuilder_metrics"` + RemoteWrite []struct { + Name string `json:"name"` + Url string `json:"url"` + } `json:"remote_write"` + } `json:"prometheus"` + ProcessExporter struct { + ProcessNames []struct { + Name string `json:"name"` + Cmdline []string `json:"cmdline"` + } `json:"process_names"` + } `json:"process_exporter"` + Fluentbit struct { + InputTags string `json:"input_tags"` + OutputCwLogGroupName string `json:"output_cw_log_group_name"` + OutputCwLogStreamPrefix string `json:"output_cw_log_stream_prefix"` + } `json:"fluentbit"` +} + +func TestMerge(t *testing.T) { + exStr := `{ + "reth_chain_downloader": { + "aws_access_key_id": "string", + "aws_secret_access_key": "string", + "rclone_s3_endpoint": "string" + }, + "orderflow_proxy": { + "flashbots_of_signing_key": "0x00", + "builder_public_ip": "1.2.3.4", + "tls_hosts": [ + "1.2.3.4", + "fundomain.builderx.io", + "172.27.14.1", + "2001:db8::123.123.123.123" + ] + }, + "rbuilder": { + "extra_data": "Illuminate Dmocratize Dstribute", + "relay_secret_key": "0x00", + "optimistic_relay_secret_key": "0x00", + "coinbase_secret_key": "0x00", + "always_seal": true, + "relays": [ + { + "name": "flashbots", + "url": "https://0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae@boost-relay.flashbots.net", + "use_ssz_for_submit": true, + "use_gzip_for_submit": false, + "priority": 0, + "optimistic": false + }, + { + "name": "ultrasound", + "url": "https://0xa1559ace749633b997cb3fdacffb890aeebdb0f5a3b6aaa7eeeaf1a38af0a8fe88b9e4b1f61f236d2e64d95733327a62@relay.ultrasound.money", + "use_ssz_for_submit": true, + "use_gzip_for_submit": true, + "priority": 1, + "optimistic": true + } + ] + }, + "prometheus": { + "scrape_interval": "10s", + "static_configs_default_labels": [ + { + "label_key": "flashbots_net_vendor", + "label_value": "azure" + }, + { + "label_key": "flashbots_net_chain", + "label_value": "mainnet" + } + ], + "lighthouse_metrics": { + "enabled": true, + "targets": [ + "localhost:5054" + ] + }, + "reth_metrics": { + "enabled": true, + "targets": [ + "localhost:9001" + ] + }, + "rbuilder_metrics": { + "enabled": true, + "targets": [ + "localhost:6069" + ] + }, + "remote_write": [ + { + "name": "tdx-rbuilder-collector", + "url": "https://aps-workspaces.us-east-2.amazonaws.com/workspaces/ws-xxx/api/v1/remote_write" + } + ] + }, + "process_exporter": { + "process_names": [ + { + "name": "lighthouse", + "cmdline": [ + "^\\/([-.0-9a-zA-Z]+\\/)*lighthouse[-.0-9a-zA-Z]* " + ] + }, + { + "name": "rbuilder", + "cmdline": [ + "^\\/([-.0-9a-zA-Z]+\\/)*rbuilder[-.0-9a-zA-Z]* " + ] + }, + { + "name": "reth", + "cmdline": [ + "^\\/([-.0-9a-zA-Z]+\\/)*reth[-.0-9a-zA-Z]* " + ] + } + ] + }, + "fluentbit": { + "input_tags": "tag-1 tag-2", + "output_cw_log_group_name": "multioperator-builder", + "output_cw_log_stream_prefix": "builder-01-" + } +} +` + secrets := make(map[string]string) + secrets["orderflow_proxy.flashbots_of_signing_key"] = "test_value_1" + newC, err := MergeConfigSecrets([]byte(exStr), secrets) + require.NoError(t, err) + + cfg := ExampleConfig{} + err = json.Unmarshal(newC, &cfg) + require.Equal(t, cfg.OrderflowProxy.FlashbotsOfSigningKey, "test_value_1") +} diff --git a/application/config_utils.go b/application/config_utils.go new file mode 100644 index 0000000..baa2c00 --- /dev/null +++ b/application/config_utils.go @@ -0,0 +1,35 @@ +package application + +import ( + "encoding/json" + "strings" + + "github.com/buger/jsonparser" +) + +func MergeConfigSecrets(config json.RawMessage, secrets map[string]string) (json.RawMessage, error) { + // merge config and secrets + var bts = []byte(config) + var err error + for k, v := range secrets { + tV := "\"" + v + "\"" + bts, err = jsonparser.Set(config, []byte(tV), strings.Split(k, ".")...) + if err != nil { + return nil, err + } + + } + return bts, nil +} + +func MergeSecrets(defaultSecrets map[string]string, secrets map[string]string) map[string]string { + // merge secrets + res := make(map[string]string) + for k, v := range defaultSecrets { + res[k] = v + } + for k, v := range secrets { + res[k] = v + } + return res +} diff --git a/application/service.go b/application/service.go new file mode 100644 index 0000000..3798145 --- /dev/null +++ b/application/service.go @@ -0,0 +1,54 @@ +package application + +import ( + "context" + "fmt" + "net" + + "github.com/flashbots/builder-hub/domain" +) + +type SecretAccessor interface { + GetSecretValues(secretName string) (map[string]string, error) +} + +type BuilderDataAccessor interface { + GetActiveMeasurements(ctx context.Context) ([]domain.Measurement, error) + GetActiveBuildersWithServiceCredentials(ctx context.Context) ([]domain.BuilderWithServices, error) + GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) + GetBuilderByIP(ip net.IP) (*domain.Builder, error) +} + +type BuilderHub struct { + dataAccessor BuilderDataAccessor +} + +func NewBuilderHub(dataAccessor BuilderDataAccessor) *BuilderHub { + return &BuilderHub{dataAccessor: dataAccessor} +} + +func (b *BuilderHub) GetAllowedMeasurements(ctx context.Context) ([]domain.Measurement, error) { + return b.dataAccessor.GetActiveMeasurements(ctx) +} + +func (b *BuilderHub) GetActiveBuilders(ctx context.Context) ([]domain.BuilderWithServices, error) { + return b.dataAccessor.GetActiveBuildersWithServiceCredentials(ctx) +} + +// func (b *BuilderHub) GetConfig(ctx context.Context, builderName string) []string { +// // get config according to builder +// // get secrets according to builder +// return nil +// } +func (b *BuilderHub) VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) { + _, err := b.dataAccessor.GetMeasurementByTypeAndHash(measurement.AttestationType, measurement.Hash) + if err != nil { + return nil, fmt.Errorf("failing to fetch corresponding measurement data %x %w", measurement.Hash, err) + } + builder, err := b.dataAccessor.GetBuilderByIP(ip) + if err != nil { + // TODO: might avoid logging ip though it should be ok, at least keep it for development state + return nil, fmt.Errorf("failing to fetch builder by ip %s %w", ip.String(), err) + } + return builder, nil +} diff --git a/cmd/httpserver/main.go b/cmd/httpserver/main.go index 781c22c..a1b9a2d 100644 --- a/cmd/httpserver/main.go +++ b/cmd/httpserver/main.go @@ -7,13 +7,16 @@ import ( "syscall" "time" + "github.com/flashbots/builder-hub/adapters/database" + "github.com/flashbots/builder-hub/application" "github.com/flashbots/builder-hub/common" "github.com/flashbots/builder-hub/httpserver" + "github.com/flashbots/builder-hub/ports" "github.com/google/uuid" "github.com/urfave/cli/v2" // imports as package "cli" ) -var flags []cli.Flag = []cli.Flag{ +var flags = []cli.Flag{ &cli.StringFlag{ Name: "listen-addr", Value: "127.0.0.1:8080", @@ -54,6 +57,12 @@ var flags []cli.Flag = []cli.Flag{ Value: 15, Usage: "seconds to wait in drain HTTP request", }, + &cli.StringFlag{ + Name: "postgres-dsn", + Value: "postgres://localhost:5432/postgres?sslmode=disable", + Usage: "Postgres DSN", + EnvVars: []string{"POSTGRES_DSN"}, + }, } func main() { @@ -86,6 +95,14 @@ func main() { RequestHeaders: true, Tags: logTags, }) + db, err := database.NewDatabaseService(cCtx.String("postgres-dsn")) + if err != nil { + log.Error("failed to create database", "err", err) + return err + } + defer db.Close() + builderHub := application.NewBuilderHub(db) + builderHandler := ports.NewBuilderHubHandler(builderHub, log) cfg := &httpserver.HTTPServerConfig{ ListenAddr: listenAddr, @@ -99,7 +116,7 @@ func main() { WriteTimeout: 30 * time.Second, } - srv, err := httpserver.NewHTTPServer(cfg) + srv, err := httpserver.NewHTTPServer(cfg, builderHandler) if err != nil { cfg.Log.Error("failed to create server", "err", err) return err diff --git a/database/database.go b/database/database.go deleted file mode 100644 index e57da45..0000000 --- a/database/database.go +++ /dev/null @@ -1,54 +0,0 @@ -// Package database exposes the postgres database -package database - -import ( - "os" - - "github.com/flashbots/builder-hub/database/migrations" - "github.com/flashbots/builder-hub/database/vars" - "github.com/jmoiron/sqlx" - _ "github.com/lib/pq" - migrate "github.com/rubenv/sql-migrate" -) - -type DatabaseService struct { - DB *sqlx.DB -} - -func NewDatabaseService(dsn string) (*DatabaseService, error) { - db, err := sqlx.Connect("postgres", dsn) - if err != nil { - return nil, err - } - - db.DB.SetMaxOpenConns(50) - db.DB.SetMaxIdleConns(10) - db.DB.SetConnMaxIdleTime(0) - - if os.Getenv("DB_DONT_APPLY_SCHEMA") == "" { - migrate.SetTable(vars.TableMigrations) - _, err := migrate.Exec(db.DB, "postgres", migrations.Migrations, migrate.Up) - if err != nil { - return nil, err - } - } - - dbService := &DatabaseService{DB: db} //nolint:exhaustruct - err = dbService.prepareNamedQueries() - return dbService, err -} - -func (s *DatabaseService) prepareNamedQueries() (err error) { - return nil -} - -func (s *DatabaseService) Close() error { - return s.DB.Close() -} - -func (s *DatabaseService) SomeQuery() (count uint64, err error) { - query := `SELECT COUNT(*) FROM ` + vars.TableTest + `;` - row := s.DB.QueryRow(query) - err = row.Scan(&count) - return count, err -} diff --git a/database/database_test.go b/database/database_test.go deleted file mode 100644 index 2aa09fe..0000000 --- a/database/database_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package database - -import ( - "os" - "testing" - - "github.com/flashbots/builder-hub/common" - "github.com/flashbots/builder-hub/database/migrations" - "github.com/flashbots/builder-hub/database/vars" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" -) - -var ( - runDBTests = os.Getenv("RUN_DB_TESTS") == "1" //|| true - testDBDSN = common.GetEnv("TEST_DB_DSN", "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable") -) - -func resetDatabase(t *testing.T) *DatabaseService { - t.Helper() - if !runDBTests { - t.Skip("Skipping database tests") - } - - // Wipe test database - _db, err := sqlx.Connect("postgres", testDBDSN) - require.NoError(t, err) - _, err = _db.Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`) - require.NoError(t, err) - - db, err := NewDatabaseService(testDBDSN) - require.NoError(t, err) - return db -} - -func TestMigrations(t *testing.T) { - db := resetDatabase(t) - query := `SELECT COUNT(*) FROM ` + vars.TableMigrations + `;` - rowCount := 0 - err := db.DB.QueryRow(query).Scan(&rowCount) - require.NoError(t, err) - require.Len(t, migrations.Migrations.Migrations, rowCount) -} - -func Test_DB1(t *testing.T) { - db := resetDatabase(t) - x, err := db.SomeQuery() - require.NoError(t, err) - require.Equal(t, uint64(0), x) -} diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go deleted file mode 100644 index a779d91..0000000 --- a/database/migrations/001_init_database.go +++ /dev/null @@ -1,21 +0,0 @@ -package migrations - -import ( - "github.com/flashbots/builder-hub/database/vars" - migrate "github.com/rubenv/sql-migrate" -) - -var Migration001InitDatabase = &migrate.Migration{ - Id: "001-init-database", - Up: []string{` - CREATE TABLE IF NOT EXISTS ` + vars.TableTest + ` ( - id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - inserted_at timestamp NOT NULL default current_timestamp - ); - `}, - Down: []string{` - DROP TABLE IF EXISTS ` + vars.TableTest + `; - `}, - DisableTransactionUp: false, - DisableTransactionDown: false, -} diff --git a/database/migrations/migration.go b/database/migrations/migration.go deleted file mode 100644 index a990cca..0000000 --- a/database/migrations/migration.go +++ /dev/null @@ -1,12 +0,0 @@ -// Package migrations contains all the migration files -package migrations - -import ( - migrate "github.com/rubenv/sql-migrate" -) - -var Migrations = migrate.MemoryMigrationSource{ - Migrations: []*migrate.Migration{ - Migration001InitDatabase, - }, -} diff --git a/database/types.go b/database/types.go deleted file mode 100644 index 85b6593..0000000 --- a/database/types.go +++ /dev/null @@ -1,29 +0,0 @@ -package database - -import ( - "database/sql" - "time" -) - -func NewNullInt64(i int64) sql.NullInt64 { - return sql.NullInt64{ - Int64: i, - Valid: true, - } -} - -func NewNullString(s string) sql.NullString { - return sql.NullString{ - String: s, - Valid: true, - } -} - -// NewNullTime returns a sql.NullTime with the given time.Time. If the time is -// the zero value, the NullTime is invalid. -func NewNullTime(t time.Time) sql.NullTime { - return sql.NullTime{ - Time: t, - Valid: t != time.Time{}, - } -} diff --git a/database/types_test.go b/database/types_test.go deleted file mode 100644 index 8f4dfed..0000000 --- a/database/types_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package database - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestNewNullTime(t *testing.T) { - var t1 time.Time - nt1 := NewNullTime(t1) - require.False(t, nt1.Valid) - - t1 = time.Now() - nt1 = NewNullTime(t1) - require.True(t, nt1.Valid) - require.Equal(t, t1, nt1.Time) -} diff --git a/database/vars/tables.go b/database/vars/tables.go deleted file mode 100644 index a1e67a7..0000000 --- a/database/vars/tables.go +++ /dev/null @@ -1,11 +0,0 @@ -// Package vars contains the database variables such as dynamic table names -package vars - -import "github.com/flashbots/builder-hub/common" - -var ( - tablePrefix = common.GetEnv("DB_TABLE_PREFIX", "dev") - - TableMigrations = tablePrefix + "_migrations" - TableTest = tablePrefix + "_test" -) diff --git a/domain/measurements-jq.json b/domain/measurements-jq.json new file mode 100644 index 0000000..f04c61c --- /dev/null +++ b/domain/measurements-jq.json @@ -0,0 +1 @@ +{"key1":"value1","key2":"value2"} diff --git a/domain/measurements.json b/domain/measurements.json new file mode 100644 index 0000000..2e51740 --- /dev/null +++ b/domain/measurements.json @@ -0,0 +1 @@ +{"1":"2214214214214","3":"217429174987412","2": "2124214"} \ No newline at end of file diff --git a/domain/types.go b/domain/types.go new file mode 100644 index 0000000..6ed88f1 --- /dev/null +++ b/domain/types.go @@ -0,0 +1,55 @@ +package domain + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net" +) + +var ( + ErrNotFound = fmt.Errorf("not found") + ErrIncorrectBuilder = fmt.Errorf("incorrect builder") +) + +type Measurement struct { + Hash []byte + AttestationType string + Measurement map[string]string +} + +// CalculateHash calculates the sha256 hash of the given measurements +// cat measurements.json | jq --sort-keys --compact-output --join-output| sha256sum +func CalculateHash(measurements map[string]string) []byte { + bts, _ := json.Marshal(measurements) + resp := sha256.Sum256(bts) + return resp[:] +} + +func NewMeasurement(attestationType string, measurements map[string]string) *Measurement { + return &Measurement{ + AttestationType: attestationType, + Measurement: measurements, + Hash: CalculateHash(measurements), + } +} + +type Builder struct { + Name string `json:"name"` + IPAddress net.IP `json:"ip_address"` + IsActive bool `json:"is_active"` +} + +type BuilderWithServices struct { + Builder Builder + Services []BuilderServices +} + +type BuilderServices struct { + TLSCert string + ECDSAPubKey []byte + Service string +} + +type BuilderConfig struct { +} diff --git a/domain/types_test.go b/domain/types_test.go new file mode 100644 index 0000000..a86c7c3 --- /dev/null +++ b/domain/types_test.go @@ -0,0 +1,29 @@ +package domain + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCalculateHash(t *testing.T) { + t.Run("simple", func(t *testing.T) { + measurements := map[string]string{ + "key1": "value1", + "key2": "value2", + } + hash := CalculateHash(measurements) + require.Equal(t, fmt.Sprintf("%x", hash), "b734413c644ec49f6a7c07d88b267244582d6422d89eee955511f6b3c0dcb0f2") + }) + t.Run("unordered", func(t *testing.T) { + measurements := map[string]string{ + "1": "2214214214214", + "3": "217429174987412", + "2": "2124214", + } + hash := CalculateHash(measurements) + require.Equal(t, fmt.Sprintf("%x", hash), "4e5f14b1f6f1a1cbff43a08b2a2f76b261e98fa8c43b51c57b120923a5f10462") + }) + +} diff --git a/go.mod b/go.mod index 0c6167b..30543d8 100644 --- a/go.mod +++ b/go.mod @@ -16,9 +16,15 @@ require ( ) require ( + github.com/aws/aws-sdk-go v1.55.5 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgtype v1.14.3 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 297bd50..a2a29ff 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,20 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/VictoriaMetrics/metrics v1.35.1 h1:o84wtBKQbzLdDy14XeskkCZih6anG+veZ1SwJHFGwrU= github.com/VictoriaMetrics/metrics v1.35.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= @@ -12,30 +23,121 @@ github.com/go-chi/httplog/v2 v2.1.1 h1:ojojiu4PIaoeJ/qAO4GWUxJqvYUTobeo7zmuHQJAx github.com/go-chi/httplog/v2 v2.1.1/go.mod h1:/XXdxicJsp4BA5fapgIC3VuTD+z0Z/VzukoB3VDc1YE= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.3 h1:h6W9cPuHsRWQFTWUZMAKMgG5jSwQI0Zurzdvlx3Plus= +github.com/jackc/pgtype v1.14.3/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= @@ -46,12 +148,109 @@ github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OL github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/httpserver.dockerfile b/httpserver.dockerfile deleted file mode 100644 index 0fed6bb..0000000 --- a/httpserver.dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM golang:1.21 AS builder -ARG VERSION -WORKDIR /build -ADD go.mod /build/ -RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOOS=linux \ - go mod download -ADD . /build/ -RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOOS=linux \ - go build \ - -trimpath \ - -ldflags "-s -X main.version=${VERSION}" \ - -v \ - -o your-project \ - cmd/httpserver/main.go - -FROM alpine:latest -WORKDIR /app -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=builder /build/your-project /app/your-project -ENV LISTEN_ADDR=":8080" -EXPOSE 8080 -CMD ["/app/your-project"] diff --git a/httpserver/server.go b/httpserver/server.go index cf7038f..099de0f 100644 --- a/httpserver/server.go +++ b/httpserver/server.go @@ -8,6 +8,7 @@ import ( "time" "github.com/flashbots/builder-hub/metrics" + "github.com/flashbots/builder-hub/ports" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httplog/v2" @@ -27,9 +28,10 @@ type HTTPServerConfig struct { } type Server struct { - cfg *HTTPServerConfig - isReady atomic.Bool - log *httplog.Logger + cfg *HTTPServerConfig + isReady atomic.Bool + log *httplog.Logger + appHandler *ports.BuilderHubHandler srv *http.Server metricsSrv *metrics.MetricsServer @@ -39,10 +41,11 @@ type Server struct { mockGetMeasurementsResponse string } -func NewHTTPServer(cfg *HTTPServerConfig) (srv *Server, err error) { +func NewHTTPServer(cfg *HTTPServerConfig, appHandler *ports.BuilderHubHandler) (srv *Server, err error) { srv = &Server{ cfg: cfg, log: cfg.Log, + appHandler: appHandler, srv: nil, metricsSrv: metrics.NewMetricsServer(cfg.MetricsAddr, nil), } @@ -74,10 +77,11 @@ func (srv *Server) getRouter() http.Handler { mux.Get("/test-panic", srv.handleTestPanic) // BuilderConfigHub API: https://www.notion.so/flashbots/BuilderConfigHub-1076b4a0d8768074bcdcd1c06c26ec87?pvs=4#10a6b4a0d87680fd81e0cad9bac3b8c5 - mux.Get("/api/l1-builder/v1/measurements", srv.handleGetMeasurements) - mux.Get("/api/l1-builder/v1/configuration", srv.handleGetConfiguration) - mux.Get("/api/l1-builder/v1/builders", srv.handleGetBuilders) - mux.Post("/api/l1-builder/v1/register_credentials/{service}", srv.handleRegisterCredentials) + mux.Get("/api/l1-builder/v1/measurements", srv.appHandler.GetAllowedMeasurements) + mux.Get("/api/l1-builder/v1/configuration", srv.appHandler.GetConfigSecrets) + mux.Get("/api/l1-builder/v1/builders", srv.appHandler.GetActiveBuilders) + mux.Post("/api/l1-builder/v1/register_credentials/{service}", srv.appHandler.RegisterCredentials) + mux.Get("/api/internal/l1-builder/v1/builders", srv.appHandler.GetActiveBuildersNoAuth) if srv.cfg.EnablePprof { srv.log.Info("pprof API enabled") diff --git a/ports/http_server.go b/ports/http_server.go new file mode 100644 index 0000000..a5d215d --- /dev/null +++ b/ports/http_server.go @@ -0,0 +1,150 @@ +package ports + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + + "github.com/flashbots/builder-hub/domain" + "github.com/go-chi/httplog/v2" +) + +type BuilderHubService interface { + GetAllowedMeasurements(ctx context.Context) ([]domain.Measurement, error) + GetActiveBuilders(ctx context.Context) ([]domain.BuilderWithServices, error) + VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) +} +type BuilderHubHandler struct { + builderHubService BuilderHubService + log *httplog.Logger +} + +func NewBuilderHubHandler(builderHubService BuilderHubService, log *httplog.Logger) *BuilderHubHandler { + return &BuilderHubHandler{builderHubService: builderHubService, log: log} +} + +type AuthData struct { + AttestationType string + MeasurementData map[string]string + IP net.IP +} + +func (bhs *BuilderHubHandler) getAuthData(r *http.Request) (*AuthData, error) { + attestationType := r.Header.Get(AttestationTypeHeader) + if attestationType == "" { + return nil, fmt.Errorf("attestation type is empty %w", ErrInvalidAuthData) + } + measurementHeader := r.Header.Get(MeasurementHeader) + measurementData := make(map[string]string) + err := json.Unmarshal([]byte(measurementHeader), &measurementData) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal measurement header %w", ErrInvalidAuthData) + } + ipHeader := r.Header.Get(ForwardedHeader) + ip := net.ParseIP(ipHeader) + if ip == nil { + return nil, fmt.Errorf("failed to parse ip %w", ErrInvalidAuthData) + } + + return &AuthData{ + AttestationType: attestationType, + MeasurementData: measurementData, + IP: ip, + }, nil +} +func (bhs *BuilderHubHandler) GetAllowedMeasurements(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + measurements, err := bhs.builderHubService.GetAllowedMeasurements(r.Context()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + var pMeasurements []Measurement + for _, m := range measurements { + pMeasurements = append(pMeasurements, fromDomainMeasurement(&m)) + } + + btsM, err := json.Marshal(measurements) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = w.Write(btsM) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func (bhs *BuilderHubHandler) GetActiveBuilders(w http.ResponseWriter, r *http.Request) { + //r.Header.Get("Authorization") + authData, err := bhs.getAuthData(r) + if err != nil { + bhs.log.Warn("malformed auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + _, err = bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, domain.NewMeasurement(authData.AttestationType, authData.MeasurementData)) + if errors.Is(err, domain.ErrNotFound) { + bhs.log.Warn("invalid auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + if err != nil { + bhs.log.Error("failed to verify ip and measurements", "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + builders, err := bhs.builderHubService.GetActiveBuilders(r.Context()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + var pBuilders []BuilderWithServiceCreds + for _, b := range builders { + pBuilders = append(pBuilders, fromDomainBuilderWithServices(&b)) + } + bts, err := json.Marshal(pBuilders) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = w.Write(bts) +} + +func (bhs *BuilderHubHandler) GetActiveBuildersNoAuth(w http.ResponseWriter, r *http.Request) { + //bhs.builderHubService + builders, err := bhs.builderHubService.GetActiveBuilders(r.Context()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + var pBuilders []BuilderWithServiceCreds + for _, b := range builders { + pBuilders = append(pBuilders, fromDomainBuilderWithServices(&b)) + } + bts, err := json.Marshal(pBuilders) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = w.Write(bts) +} + +func (bhs *BuilderHubHandler) GetConfigSecrets(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + +func (bhs *BuilderHubHandler) RegisterCredentials(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} diff --git a/ports/types.go b/ports/types.go new file mode 100644 index 0000000..8ca8a72 --- /dev/null +++ b/ports/types.go @@ -0,0 +1,77 @@ +package ports + +import ( + "encoding/hex" + "encoding/json" + "errors" + + "github.com/flashbots/builder-hub/domain" +) + +const ( + AttestationTypeHeader string = "X-Flashbots-Attestation-Type" + MeasurementHeader string = "X-Flashbots-Measurement" + ForwardedHeader string = "X-Forwarded-For" +) + +var ( + ErrInvalidAuthData = errors.New("invalid auth data") +) + +type BuilderWithServiceCreds struct { + Ip string + ServiceCreds map[string]ServiceCred +} + +type ServiceCred struct { + TlsCert string `json:"tls_cert,omitempty"` + EcdsaPubkey string `json:"ecdsa_pubkey,omitempty"` +} + +// MarshalJSON is a custom json marshaller. Unfortunately, there seems to be no way to inline map[string]Service when marshalling +// so we need to be careful when adding new fields, since custom json implementation will ignore it by default +func (b BuilderWithServiceCreds) MarshalJSON() ([]byte, error) { + // Create a map to hold all fields + m := make(map[string]interface{}) + + // Add the Ip field + m["ip"] = b.Ip + + // Add all services + for k, v := range b.ServiceCreds { + m[k] = v + } + + // Marshal the map + return json.Marshal(m) +} + +func fromDomainBuilderWithServices(builder *domain.BuilderWithServices) BuilderWithServiceCreds { + b := BuilderWithServiceCreds{} + + b.Ip = builder.Builder.IPAddress.String() + b.ServiceCreds = make(map[string]ServiceCred) + for _, v := range builder.Services { + b.ServiceCreds[v.Service] = ServiceCred{ + TlsCert: v.TLSCert, + EcdsaPubkey: hex.EncodeToString(v.ECDSAPubKey), + } + } + + return b +} + +type Measurement struct { + Hash string `json:"measurement_hash"` + AttestationType string `json:"attestation_type"` + Measurement map[string]string `json:"measurement"` +} + +func fromDomainMeasurement(measurement *domain.Measurement) Measurement { + m := Measurement{ + Hash: hex.EncodeToString(measurement.Hash), + AttestationType: measurement.AttestationType, + Measurement: measurement.Measurement, + } + return m +} diff --git a/ports/types_test.go b/ports/types_test.go new file mode 100644 index 0000000..9844a60 --- /dev/null +++ b/ports/types_test.go @@ -0,0 +1,27 @@ +package ports + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestJson(t *testing.T) { + b := BuilderWithServiceCreds{ + Ip: "1.2.3.4", + ServiceCreds: map[string]Service{ + "of-proxy": { + TlsCert: "cert", + EcdsaPubkey: "pubkey", + }, + "rbuilder": { + TlsCert: "cert", + EcdsaPubkey: "pubkey", + }, + }, + } + bts, err := json.Marshal(b) + require.NoError(t, err) + require.Equal(t, string(bts), `{"ip":"1.2.3.4","of-proxy":{"tls_cert":"cert","ecdsa_pubkey":"pubkey"},"rbuilder":{"tls_cert":"cert","ecdsa_pubkey":"pubkey"}}`) +} diff --git a/schema/000_init.sql b/schema/000_init.sql new file mode 100644 index 0000000..7d00e7f --- /dev/null +++ b/schema/000_init.sql @@ -0,0 +1,97 @@ +CREATE TABLE builders ( + name VARCHAR(255) PRIMARY KEY, + ip_address INET NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + deprecated_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT active_only_when_not_deprecated CHECK ( + (is_active = true AND deprecated_at IS NULL) OR + (is_active = false) + ) +); + +-- Add an index on ip_address for faster lookups +CREATE INDEX idx_builders_ip_address ON builders(ip_address); + +-- Trigger to automatically update the updated_at timestamp +CREATE OR REPLACE FUNCTION update_builders_updated_at() + RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trigger_update_builders_updated_at + BEFORE UPDATE ON builders + FOR EACH ROW +EXECUTE FUNCTION update_builders_updated_at(); + +-- Measurements Whitelist table +CREATE TABLE measurements_whitelist ( + hash BYTEA PRIMARY KEY , -- hash of the measurement + attestation_type TEXT NOT NULL, -- attestation type of the measurement + measurement JSONB NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT true, + + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + deprecated_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT active_only_when_not_deprecated CHECK ( + (is_active = true AND deprecated_at IS NULL) OR + (is_active = false) + ) +); + + + +CREATE TABLE service_credential_registrations ( + id SERIAL PRIMARY KEY, + builder_name VARCHAR(255) REFERENCES builders(name), + service TEXT NOT NULL, + tls_cert TEXT, + ecdsa_pubkey BYTEA, + is_active BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX idx_unique_active_credential_per_builder_service + ON service_credential_registrations (builder_name, service) + WHERE is_active = true; + +-- Trigger to automatically update the updated_at timestamp +CREATE OR REPLACE FUNCTION update_service_credential_registrations_updated_at() + RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trigger_update_service_credential_registrations_updated_at + BEFORE UPDATE ON service_credential_registrations + FOR EACH ROW +EXECUTE FUNCTION update_service_credential_registrations_updated_at(); + +-- Updated builder_configs table +CREATE TABLE builder_configs ( + id SERIAL PRIMARY KEY, + builder_name VARCHAR(255) REFERENCES builders(name), -- references name + config JSONB NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + + + +-- Add a new constraint to ensure only one active config per builder +ALTER TABLE builder_configs + ADD CONSTRAINT unique_active_config_per_builder + EXCLUDE (builder_name WITH =) + WHERE (is_active = true); + + + From 7c75943c84eb760f7f909ea000e675103fcb66c4 Mon Sep 17 00:00:00 2001 From: Daniel Sukoneck Date: Mon, 7 Oct 2024 17:14:47 -0700 Subject: [PATCH 2/6] add testdata to dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a231157..1bb5d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,5 @@ RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 WORKDIR /app COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /build/builder-hub /app/builder-hub +ADD testdata/ /app/testdata/ CMD ["/app/builder-hub"] From 74153c6e0beef0e30199a5a739cfc378fb41c653 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 8 Oct 2024 16:40:11 +0200 Subject: [PATCH 3/6] initial implementation draft --- adapters/database/service.go | 231 ++++------------------ adapters/database/types.go | 4 +- adapters/secrets/service.go | 41 +++- application/config_utils.go | 14 +- application/service.go | 43 ++-- cmd/httpserver/main.go | 28 ++- domain/types.go | 10 +- domain/types_test.go | 49 ++++- httpserver/handler_test.go | 5 +- ports/{http_server.go => http_handler.go} | 101 +++++++++- ports/types.go | 6 +- ports/types_test.go | 27 --- schema/000_init.sql | 117 ++++++----- 13 files changed, 343 insertions(+), 333 deletions(-) rename ports/{http_server.go => http_handler.go} (55%) delete mode 100644 ports/types_test.go diff --git a/adapters/database/service.go b/adapters/database/service.go index 30f28c7..313121b 100644 --- a/adapters/database/service.go +++ b/adapters/database/service.go @@ -3,7 +3,9 @@ package database import ( "context" "database/sql" + "encoding/json" "errors" + "fmt" "net" "github.com/flashbots/builder-hub/domain" @@ -34,111 +36,7 @@ func (s *Service) Close() error { return s.DB.Close() } -// -//// GetAllowedMeasurements retrieves all active measurements -//func (s *Service) GetAllowedMeasurements() ([]Measurement, error) { -// var measurements []Measurement -// err := s.DB.Select(&measurements, ` -// SELECT id, measurement -// FROM measurements_whitelist -// WHERE is_active = true -// `) -// return measurements, err -//} -// -//// GetActiveBuilders retrieves all active builders with their IPs and public keys -//func (s *Service) GetActiveBuilders() ([]ActiveBuilder, error) { -// var builders []ActiveBuilder -// err := s.DB.Select(&builders, ` -// SELECT b.id AS builder_id, b.name, iw.ip_address, scr.tls_pubkey, scr.ecdsa_pubkey -// FROM builders b -// JOIN ip_whitelist iw ON b.id = iw.builder_id -// JOIN service_credential_registrations scr ON iw.id = scr.ip_whitelist_id -// WHERE iw.is_active = true AND scr.service = 'builder' -// `) -// return builders, err -//} -// -//// GetConfigByIP retrieves the config for a builder based on their IP address -//func (s *Service) GetConfigByIP(ip string) (BuilderConfig, error) { -// var config BuilderConfig -// err := s.DB.Get(&config, ` -// SELECT bc.builder_id, bc.config -// FROM builder_configs bc -// JOIN ip_whitelist iw ON bc.builder_id = iw.builder_id -// WHERE iw.ip_address = $1 AND iw.is_active = true AND bc.is_active = true -// `, ip) -// return config, err -//} -// -//// RegisterNewCredentials registers new credentials for a service -//func (s *Service) RegisterNewCredentials(ip, service string, tlsPubKey, ecdsaPubKey []byte) error { -// _, err := s.DB.Exec(` -// INSERT INTO service_credential_registrations (ip_whitelist_id, service, tls_pubkey, ecdsa_pubkey) -// SELECT id, $2, $3, $4 -// FROM ip_whitelist -// WHERE ip_address = $1 AND is_active = true -// `, ip, service, tlsPubKey, ecdsaPubKey) -// return err -//} -// -//// GetBuilderByIP retrieves an active builder by their IP address -//func (s *Service) GetActiveBuilderByIP(ip net.IP) (*ActiveBuilder, error) { -// var paramIP pgtype.Inet -// err := paramIP.Set(ip) -// if err != nil { -// return nil, err -// } -// -// var builder ActiveBuilder -// err = s.DB.Get(&builder, ` -// SELECT b.id AS builder_id, b.name, iw.ip_address, scr.tls_pubkey, scr.ecdsa_pubkey -// FROM builders b -// JOIN ip_whitelist iw ON b.id = iw.builder_id -// JOIN service_credential_registrations scr ON iw.id = scr.ip_whitelist_id -// WHERE iw.ip_address = $1 AND iw.is_active = true AND scr.service = 'builder' -// `, paramIP) -// -// if err != nil { -// return nil, fmt.Errorf("error getting builder by IP: %w", err) -// } -// -// return &builder, nil -//} -// -//type IPWhitelistEntry struct { -// BuilderID int `db:"builder_id"` -// IPAddress pgtype.Inet `db:"ip_address"` -// IsActive bool `db:"is_active"` -// ValidFrom time.Time `db:"valid_from"` -// ValidTo *time.Time `db:"valid_to"` -//} -// -//// use for validation if needed -//func (s *Service) GetIPWhitelistByIP(ip net.IP) (*IPWhitelistEntry, error) { -// -// var paramIP pgtype.Inet -// err := paramIP.Set(ip) -// if err != nil { -// return nil, err -// } -// -// var entry IPWhitelistEntry -// err = s.DB.Get(&entry, ` -// SELECT builder_id, ip_address, is_active, valid_from, valid_to -// FROM ip_whitelist -// WHERE ip_address = $1 -// `, paramIP) -// -// if err != nil { -// return nil, fmt.Errorf("error getting IP whitelist entry: %w", err) -// } -// -// //entry.IPAddress.IPNet.IP -// return &entry, nil -//} - -// GetMeasurementByOIDAndHash retrieves a measurement by OID and hash +// GetMeasurementByTypeAndHash retrieves a measurement by OID and hash func (s *Service) GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) { var m Measurement err := s.DB.Get(&m, ` @@ -188,20 +86,11 @@ func (s *Service) GetActiveMeasurements(ctx context.Context) ([]domain.Measureme return domainMeasurements, err } -// GetActiveBuilders retrieves all active builders -func (s *Service) GetActiveBuilders() ([]Builder, error) { - var builders []Builder - err := s.DB.Select(&builders, ` - SELECT * FROM builders - WHERE is_active = true AND deprecated_at IS NULL - `) - return builders, err -} - // RegisterCredentialsForBuilder registers new credentials for a builder, deprecating all previous credentials -func (s *Service) RegisterCredentialsForBuilder(builderName, service, tlsCert string, ecdsaPubKey []byte) error { +// It uses hash and attestation_type to fetch the corresponding measurement_id via a subquery. +func (s *Service) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error { // Start a transaction - tx, err := s.DB.Begin() + tx, err := s.DB.BeginTx(ctx, nil) if err != nil { return err } @@ -210,14 +99,14 @@ func (s *Service) RegisterCredentialsForBuilder(builderName, service, tlsCert st // Deprecate all previous credentials for this builder and service _, err = tx.Exec(` UPDATE service_credential_registrations - SET is_active = false + SET is_active = false, deprecated_at = NOW() WHERE builder_name = $1 AND service = $2 `, builderName, service) if err != nil { return err } - // Insert new credentials + // Insert new credentials with a subquery to fetch the measurement_id var nullableTLSCert sql.NullString if tlsCert != "" { nullableTLSCert = sql.NullString{String: tlsCert, Valid: true} @@ -225,11 +114,13 @@ func (s *Service) RegisterCredentialsForBuilder(builderName, service, tlsCert st _, err = tx.Exec(` INSERT INTO service_credential_registrations - (builder_name, service, tls_cert, ecdsa_pubkey, is_active) - VALUES ($1, $2, $3, $4, true) - `, builderName, service, nullableTLSCert, ecdsaPubKey) + (builder_name, service, tls_cert, ecdsa_pubkey, is_active, measurement_id) + VALUES ($1, $2, $3, $4, true, + (SELECT id FROM measurements_whitelist WHERE hash = $5 AND attestation_type = $6) + ) + `, builderName, service, nullableTLSCert, ecdsaPubKey, measurementHash, attestationType) if err != nil { - return err + return fmt.Errorf("failed to insert credentials for builder %s: %w", builderName, err) } // Commit the transaction @@ -241,17 +132,17 @@ func (s *Service) RegisterCredentialsForBuilder(builderName, service, tlsCert st } // GetActiveConfigForBuilder retrieves the active config for a builder by name -func (s *Service) GetActiveConfigForBuilder(builderName string) (*BuilderConfig, error) { +func (s *Service) GetActiveConfigForBuilder(ctx context.Context, builderName string) (json.RawMessage, error) { var config BuilderConfig - err := s.DB.Get(&config, ` + err := s.DB.GetContext(ctx, &config, ` SELECT * FROM builder_configs WHERE builder_name = $1 AND is_active = true `, builderName) - return &config, err + return config.Config, err } func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ([]domain.BuilderWithServices, error) { - rows, err := s.DB.Query(` + rows, err := s.DB.QueryContext(ctx, ` SELECT b.name, b.ip_address, @@ -276,7 +167,8 @@ func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ( for rows.Next() { var ipAddress pgtype.Inet - var builderName, service string + var builderName string + var service sql.NullString var tlsCert sql.NullString var ecdsaPubKey []byte @@ -294,9 +186,9 @@ func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ( buildersMap[builderName] = builder } - if service != "" { + if service.Valid { builder.Credentials = append(builder.Credentials, ServiceCredential{ - Service: service, + Service: service.String, TLSCert: tlsCert, ECDSAPubKey: ecdsaPubKey, }) @@ -320,65 +212,20 @@ func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ( return builders, nil } -// -//// GetActiveBuildersWithCredentials retrieves all active builders along with their active service credentials -//func (s *Service) GetActiveBuildersServices(ctx context.Context) ([]domain.BuilderWithServices, error) { -// rows, err := s.DB.QueryContext(ctx, ` -// SELECT -// b.name, -// b.ip_address, -// json_agg( -// json_build_object( -// 'service', scr.service, -// 'tls_cert', scr.tls_cert, -// 'ecdsa_pubkey', encode(scr.ecdsa_pubkey, 'base64') -// ) -// ) AS credentials -// FROM -// builders b -// LEFT JOIN -// service_credential_registrations scr ON b.name = scr.builder_name -// WHERE -// b.is_active = true AND scr.is_active = true -// GROUP BY -// b.name, b.ip_address -// `) -// if err != nil { -// return nil, err -// } -// defer rows.Close() -// -// var builders []BuilderWithCredentials -// for rows.Next() { -// var builder BuilderWithCredentials -// var credentialsJSON []byte -// err := rows.Scan(&builder.Name, &builder.IPAddress, &credentialsJSON) -// if err != nil { -// return nil, err -// } -// -// if credentialsJSON != nil { -// err = json.Unmarshal(credentialsJSON, &builder.Credentials) -// if err != nil { -// return nil, err -// } -// } -// -// builders = append(builders, builder) -// } -// -// if err = rows.Err(); err != nil { -// return nil, err -// } -// -// var dBuilders []domain.BuilderWithServices -// for _, b := range builders { -// dBuilder, err := toDomainBuilderWithCredentials(b) -// if err != nil { -// return nil, err -// } -// dBuilders = append(dBuilders, *dBuilder) -// } -// -// return dBuilders, nil -//} +// LogEvent creates a new log entry in the event_log table. +// It uses hash and attestation_type to fetch the corresponding measurement_id via a subquery. +func (s *Service) LogEvent(ctx context.Context, eventName, builderName, hash, attestationType string) error { + // Insert new event log entry with a subquery to fetch the measurement_id + _, err := s.DB.ExecContext(ctx, ` + INSERT INTO event_log + (event_name, builder_name, measurement_id) + VALUES ($1, $2, + (SELECT id FROM measurements_whitelist WHERE hash = $3 AND attestation_type = $4) + ) + `, eventName, builderName, hash, attestationType) + if err != nil { + return fmt.Errorf("failed to insert event log for builder %s: %w", builderName, err) + } + + return nil +} diff --git a/adapters/database/types.go b/adapters/database/types.go index 66cd1b2..ca0cd24 100644 --- a/adapters/database/types.go +++ b/adapters/database/types.go @@ -18,6 +18,7 @@ type ActiveBuilder struct { } type Measurement struct { + ID int `db:"id"` Hash []byte `db:"hash"` AttestationType string `db:"attestation_type"` Measurement json.RawMessage `db:"measurement"` @@ -30,7 +31,7 @@ type Measurement struct { func convertMeasurementToDomain(measurement Measurement) (*domain.Measurement, error) { var m domain.Measurement m.AttestationType = measurement.AttestationType - m.Measurement = make(map[string]string) + m.Measurement = make(map[string]domain.SingleMeasurement) err := json.Unmarshal(measurement.Measurement, &m.Measurement) if err != nil { return nil, err @@ -104,6 +105,7 @@ func toDomainBuilderWithCredentials(builder BuilderWithCredentials) (*domain.Bui s.Services = append(s.Services, domain.BuilderServices{ TLSCert: cred.TLSCert.String, ECDSAPubKey: cred.ECDSAPubKey, + Service: cred.Service, }) } return &s, nil diff --git a/adapters/secrets/service.go b/adapters/secrets/service.go index db1c66b..3e9821c 100644 --- a/adapters/secrets/service.go +++ b/adapters/secrets/service.go @@ -9,10 +9,11 @@ import ( ) type Service struct { - sm *secretsmanager.SecretsManager + sm *secretsmanager.SecretsManager + secretName string } -func NewService() (*Service, error) { +func NewService(secretName string) (*Service, error) { sess, err := session.NewSession(&aws.Config{ Region: aws.String("us-east-2"), }) @@ -23,12 +24,12 @@ func NewService() (*Service, error) { // Create a Secrets Manager client svc := secretsmanager.New(sess) - return &Service{sm: svc}, nil + return &Service{sm: svc, secretName: secretName}, nil } -func (s *Service) GetSecretValues(secretName string) (map[string]string, error) { +func (s *Service) GetSecretValues(builderName string) (map[string]string, error) { input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secretName), + SecretId: aws.String(s.secretName), } result, err := s.sm.GetSecretValue(input) @@ -40,5 +41,33 @@ func (s *Service) GetSecretValues(secretName string) (map[string]string, error) if err != nil { return nil, err } - return secretData, nil + defaultStr := secretData["default"] + defaultSecrets := make(map[string]string) + err = json.Unmarshal([]byte(defaultStr), &defaultSecrets) + if err != nil { + return nil, err + } + + builderStr, ok := secretData[builderName] + if !ok { + return defaultSecrets, nil + } + builderSecrets := make(map[string]string) + err = json.Unmarshal([]byte(builderStr), &builderSecrets) + if err != nil { + return nil, err + } + return MergeSecrets(defaultSecrets, builderSecrets), nil +} + +func MergeSecrets(defaultSecrets map[string]string, secrets map[string]string) map[string]string { + // merge secrets + res := make(map[string]string) + for k, v := range defaultSecrets { + res[k] = v + } + for k, v := range secrets { + res[k] = v + } + return res } diff --git a/application/config_utils.go b/application/config_utils.go index baa2c00..134ad4c 100644 --- a/application/config_utils.go +++ b/application/config_utils.go @@ -13,7 +13,7 @@ func MergeConfigSecrets(config json.RawMessage, secrets map[string]string) (json var err error for k, v := range secrets { tV := "\"" + v + "\"" - bts, err = jsonparser.Set(config, []byte(tV), strings.Split(k, ".")...) + bts, err = jsonparser.Set(bts, []byte(tV), strings.Split(k, ".")...) if err != nil { return nil, err } @@ -21,15 +21,3 @@ func MergeConfigSecrets(config json.RawMessage, secrets map[string]string) (json } return bts, nil } - -func MergeSecrets(defaultSecrets map[string]string, secrets map[string]string) map[string]string { - // merge secrets - res := make(map[string]string) - for k, v := range defaultSecrets { - res[k] = v - } - for k, v := range secrets { - res[k] = v - } - return res -} diff --git a/application/service.go b/application/service.go index 3798145..829cdd9 100644 --- a/application/service.go +++ b/application/service.go @@ -2,29 +2,33 @@ package application import ( "context" + "encoding/json" "fmt" "net" "github.com/flashbots/builder-hub/domain" ) -type SecretAccessor interface { - GetSecretValues(secretName string) (map[string]string, error) -} - type BuilderDataAccessor interface { GetActiveMeasurements(ctx context.Context) ([]domain.Measurement, error) GetActiveBuildersWithServiceCredentials(ctx context.Context) ([]domain.BuilderWithServices, error) GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) GetBuilderByIP(ip net.IP) (*domain.Builder, error) + GetActiveConfigForBuilder(ctx context.Context, builderName string) (json.RawMessage, error) + RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error +} + +type SecretAccessor interface { + GetSecretValues(builderName string) (map[string]string, error) } type BuilderHub struct { - dataAccessor BuilderDataAccessor + dataAccessor BuilderDataAccessor + secretAccessor SecretAccessor } -func NewBuilderHub(dataAccessor BuilderDataAccessor) *BuilderHub { - return &BuilderHub{dataAccessor: dataAccessor} +func NewBuilderHub(dataAccessor BuilderDataAccessor, secretAccessor SecretAccessor) *BuilderHub { + return &BuilderHub{dataAccessor: dataAccessor, secretAccessor: secretAccessor} } func (b *BuilderHub) GetAllowedMeasurements(ctx context.Context) ([]domain.Measurement, error) { @@ -35,11 +39,26 @@ func (b *BuilderHub) GetActiveBuilders(ctx context.Context) ([]domain.BuilderWit return b.dataAccessor.GetActiveBuildersWithServiceCredentials(ctx) } -// func (b *BuilderHub) GetConfig(ctx context.Context, builderName string) []string { -// // get config according to builder -// // get secrets according to builder -// return nil -// } +func (b *BuilderHub) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error { + return b.dataAccessor.RegisterCredentialsForBuilder(ctx, builderName, service, tlsCert, ecdsaPubKey, measurementHash, attestationType) +} + +func (b *BuilderHub) GetConfigWithSecrets(ctx context.Context, builderName string) ([]byte, error) { + configOpaque, err := b.dataAccessor.GetActiveConfigForBuilder(ctx, builderName) + if err != nil { + return nil, fmt.Errorf("failing to fetch config for builder %s %w", builderName, err) + } + secrets, err := b.secretAccessor.GetSecretValues(builderName) + if err != nil { + return nil, fmt.Errorf("failing to fetch secrets for builder %s %w", builderName, err) + } + res, err := MergeConfigSecrets(configOpaque, secrets) + if err != nil { + return nil, fmt.Errorf("failing to merge config and secrets %w", err) + } + return res, nil +} + func (b *BuilderHub) VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) { _, err := b.dataAccessor.GetMeasurementByTypeAndHash(measurement.AttestationType, measurement.Hash) if err != nil { diff --git a/cmd/httpserver/main.go b/cmd/httpserver/main.go index a1b9a2d..57b95c6 100644 --- a/cmd/httpserver/main.go +++ b/cmd/httpserver/main.go @@ -8,6 +8,7 @@ import ( "time" "github.com/flashbots/builder-hub/adapters/database" + "github.com/flashbots/builder-hub/adapters/secrets" "github.com/flashbots/builder-hub/application" "github.com/flashbots/builder-hub/common" "github.com/flashbots/builder-hub/httpserver" @@ -18,9 +19,10 @@ import ( var flags = []cli.Flag{ &cli.StringFlag{ - Name: "listen-addr", - Value: "127.0.0.1:8080", - Usage: "address to serve API", + Name: "listen-addr", + Value: "127.0.0.1:8080", + Usage: "address to serve API", + EnvVars: []string{"LISTEN_ADDR"}, }, &cli.StringFlag{ Name: "metrics-addr", @@ -63,6 +65,12 @@ var flags = []cli.Flag{ Usage: "Postgres DSN", EnvVars: []string{"POSTGRES_DSN"}, }, + &cli.StringFlag{ + Name: "secret-name", + Value: "", + Usage: "AWS Secret name", + EnvVars: []string{"AWS_SECRET_NAME"}, + }, } func main() { @@ -101,7 +109,13 @@ func main() { return err } defer db.Close() - builderHub := application.NewBuilderHub(db) + + sm, err := secrets.NewService(cCtx.String("secret-name")) + if err != nil { + log.Error("failed to create secrets manager", "err", err) + return err + } + builderHub := application.NewBuilderHub(db, sm) builderHandler := ports.NewBuilderHubHandler(builderHub, log) cfg := &httpserver.HTTPServerConfig{ @@ -122,12 +136,6 @@ func main() { return err } - err = srv.LoadMockResponses() - if err != nil { - cfg.Log.Error("failed to load mock responses", "err", err) - return err - } - exit := make(chan os.Signal, 1) signal.Notify(exit, os.Interrupt, syscall.SIGTERM) srv.RunInBackground() diff --git a/domain/types.go b/domain/types.go index 6ed88f1..01e6f80 100644 --- a/domain/types.go +++ b/domain/types.go @@ -15,18 +15,22 @@ var ( type Measurement struct { Hash []byte AttestationType string - Measurement map[string]string + Measurement map[string]SingleMeasurement +} + +type SingleMeasurement struct { + Expected string `json:"expected"` } // CalculateHash calculates the sha256 hash of the given measurements // cat measurements.json | jq --sort-keys --compact-output --join-output| sha256sum -func CalculateHash(measurements map[string]string) []byte { +func CalculateHash(measurements map[string]SingleMeasurement) []byte { bts, _ := json.Marshal(measurements) resp := sha256.Sum256(bts) return resp[:] } -func NewMeasurement(attestationType string, measurements map[string]string) *Measurement { +func NewMeasurement(attestationType string, measurements map[string]SingleMeasurement) *Measurement { return &Measurement{ AttestationType: attestationType, Measurement: measurements, diff --git a/domain/types_test.go b/domain/types_test.go index a86c7c3..4904874 100644 --- a/domain/types_test.go +++ b/domain/types_test.go @@ -1,6 +1,7 @@ package domain import ( + "encoding/json" "fmt" "testing" @@ -9,21 +10,51 @@ import ( func TestCalculateHash(t *testing.T) { t.Run("simple", func(t *testing.T) { - measurements := map[string]string{ - "key1": "value1", - "key2": "value2", + measurements := map[string]SingleMeasurement{ + "key1": {"value1"}, + "key2": {"value2"}, } hash := CalculateHash(measurements) - require.Equal(t, fmt.Sprintf("%x", hash), "b734413c644ec49f6a7c07d88b267244582d6422d89eee955511f6b3c0dcb0f2") + require.Equal(t, fmt.Sprintf("%x", hash), "f0605189145d385e22a5dcba5f850d6fcafe83dde8586bc79b8f2dc3450ae849") }) t.Run("unordered", func(t *testing.T) { - measurements := map[string]string{ - "1": "2214214214214", - "3": "217429174987412", - "2": "2124214", + measurements := map[string]SingleMeasurement{ + "1": {"2214214214214"}, + "3": {"217429174987412"}, + "2": {"2124214"}, } hash := CalculateHash(measurements) - require.Equal(t, fmt.Sprintf("%x", hash), "4e5f14b1f6f1a1cbff43a08b2a2f76b261e98fa8c43b51c57b120923a5f10462") + require.Equal(t, fmt.Sprintf("%x", hash), "2a4d79238e4876617c64eb3d5a46f8e851399d0f91311b8d74e2b499fd2b576d") + }) + t.Run("attestation-td", func(t *testing.T) { + measuremnt := make(map[string]SingleMeasurement) + s := `{ + "11": { + "expected": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "12": { + "expected": "f1a142c53586e7e2223ec74e5f4d1a4942956b1fd9ac78fafcdf85117aa345da" + }, + "13": { + "expected": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "15": { + "expected": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "4": { + "expected": "98ba2c602b62e67b8e0bd6c6676f12ade320a763e5e4564f62fd875a502dd651" + }, + "8": { + "expected": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "9": { + "expected": "e77938394412d83a8d4de52cdaf97df82a4d4059e1e7c4fc3c73581816cea496" + } +}` + err := json.Unmarshal([]byte(s), &measuremnt) + require.NoError(t, err) + hash := CalculateHash(measuremnt) + require.Equal(t, fmt.Sprintf("%x", hash), "538c7595f3570c9dab335a0232c84a7e7a352acd4c84f78cd2b5e70429acf30b") }) } diff --git a/httpserver/handler_test.go b/httpserver/handler_test.go index e8cfdd5..6408261 100644 --- a/httpserver/handler_test.go +++ b/httpserver/handler_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/flashbots/builder-hub/common" + "github.com/flashbots/builder-hub/ports" "github.com/go-chi/httplog/v2" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func Test_Handlers_Healthcheck_Drain_Undrain(t *testing.T) { DrainDuration: latency, ListenAddr: listenAddr, Log: getTestLogger(), - }) + }, ports.NewBuilderHubHandler(nil, getTestLogger())) require.NoError(t, err) { // Check health @@ -112,7 +113,7 @@ func Test_Handlers_BuilderConfigHub(t *testing.T) { {http.MethodPost, "/api/l1-builder/v1/register_credentials/rbuilder", []byte(`{"var1":"foo"}`)}, } - srv, err := NewHTTPServer(testServerConfig) + srv, err := NewHTTPServer(testServerConfig, ports.NewBuilderHubHandler(nil, getTestLogger())) require.NoError(t, err) for _, r := range routes { diff --git a/ports/http_server.go b/ports/http_handler.go similarity index 55% rename from ports/http_server.go rename to ports/http_handler.go index a5d215d..add174b 100644 --- a/ports/http_server.go +++ b/ports/http_handler.go @@ -2,6 +2,7 @@ package ports import ( "context" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -10,6 +11,7 @@ import ( "net/http" "github.com/flashbots/builder-hub/domain" + "github.com/go-chi/chi/v5" "github.com/go-chi/httplog/v2" ) @@ -17,6 +19,8 @@ type BuilderHubService interface { GetAllowedMeasurements(ctx context.Context) ([]domain.Measurement, error) GetActiveBuilders(ctx context.Context) ([]domain.BuilderWithServices, error) VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) + GetConfigWithSecrets(ctx context.Context, builderName string) ([]byte, error) + RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error } type BuilderHubHandler struct { builderHubService BuilderHubService @@ -29,7 +33,7 @@ func NewBuilderHubHandler(builderHubService BuilderHubService, log *httplog.Logg type AuthData struct { AttestationType string - MeasurementData map[string]string + MeasurementData map[string]domain.SingleMeasurement IP net.IP } @@ -39,7 +43,7 @@ func (bhs *BuilderHubHandler) getAuthData(r *http.Request) (*AuthData, error) { return nil, fmt.Errorf("attestation type is empty %w", ErrInvalidAuthData) } measurementHeader := r.Header.Get(MeasurementHeader) - measurementData := make(map[string]string) + measurementData := make(map[string]domain.SingleMeasurement) err := json.Unmarshal([]byte(measurementHeader), &measurementData) if err != nil { return nil, fmt.Errorf("failed to unmarshal measurement header %w", ErrInvalidAuthData) @@ -59,11 +63,13 @@ func (bhs *BuilderHubHandler) getAuthData(r *http.Request) (*AuthData, error) { func (bhs *BuilderHubHandler) GetAllowedMeasurements(w http.ResponseWriter, r *http.Request) { _, err := io.ReadAll(r.Body) if err != nil { + bhs.log.Error("failed to read request body", "error", err) w.WriteHeader(http.StatusInternalServerError) return } measurements, err := bhs.builderHubService.GetAllowedMeasurements(r.Context()) if err != nil { + bhs.log.Error("failed to fetch allowed measurements from db", "error", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -72,17 +78,16 @@ func (bhs *BuilderHubHandler) GetAllowedMeasurements(w http.ResponseWriter, r *h pMeasurements = append(pMeasurements, fromDomainMeasurement(&m)) } - btsM, err := json.Marshal(measurements) + btsM, err := json.Marshal(pMeasurements) if err != nil { + bhs.log.Error("failed to marshal measurements", "error", err) w.WriteHeader(http.StatusInternalServerError) return } _, err = w.Write(btsM) if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return + bhs.log.Error("failed to write response", "error", err) } - w.WriteHeader(http.StatusOK) } func (bhs *BuilderHubHandler) GetActiveBuilders(w http.ResponseWriter, r *http.Request) { @@ -107,6 +112,7 @@ func (bhs *BuilderHubHandler) GetActiveBuilders(w http.ResponseWriter, r *http.R builders, err := bhs.builderHubService.GetActiveBuilders(r.Context()) if err != nil { + bhs.log.Error("failed to fetch active builders from db", "error", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -116,16 +122,21 @@ func (bhs *BuilderHubHandler) GetActiveBuilders(w http.ResponseWriter, r *http.R } bts, err := json.Marshal(pBuilders) if err != nil { + bhs.log.Error("failed to marshal builders", "error", err) w.WriteHeader(http.StatusInternalServerError) return } _, err = w.Write(bts) + if err != nil { + bhs.log.Error("failed to write response", "error", err) + } } func (bhs *BuilderHubHandler) GetActiveBuildersNoAuth(w http.ResponseWriter, r *http.Request) { //bhs.builderHubService builders, err := bhs.builderHubService.GetActiveBuilders(r.Context()) if err != nil { + bhs.log.Error("failed to fetch active builders from db", "error", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -135,16 +146,90 @@ func (bhs *BuilderHubHandler) GetActiveBuildersNoAuth(w http.ResponseWriter, r * } bts, err := json.Marshal(pBuilders) if err != nil { + bhs.log.Error("failed to marshal builders", "error", err) w.WriteHeader(http.StatusInternalServerError) return } _, err = w.Write(bts) + if err != nil { + bhs.log.Error("failed to write response", "error", err) + } } func (bhs *BuilderHubHandler) GetConfigSecrets(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotImplemented) + authData, err := bhs.getAuthData(r) + if err != nil { + bhs.log.Warn("malformed auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + builder, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, domain.NewMeasurement(authData.AttestationType, authData.MeasurementData)) + if errors.Is(err, domain.ErrNotFound) { + bhs.log.Warn("invalid auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + bts, err := bhs.builderHubService.GetConfigWithSecrets(r.Context(), builder.Name) + if err != nil { + bhs.log.Error("failed to get config with secrets", "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = w.Write(bts) + if err != nil { + bhs.log.Error("failed to write response", "error", err) + } } func (bhs *BuilderHubHandler) RegisterCredentials(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotImplemented) + authData, err := bhs.getAuthData(r) + if err != nil { + bhs.log.Warn("malformed auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + measurement := domain.NewMeasurement(authData.AttestationType, authData.MeasurementData) + builder, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, measurement) + if errors.Is(err, domain.ErrNotFound) { + bhs.log.Warn("invalid auth data", "error", err) + w.WriteHeader(http.StatusForbidden) + return + } + + service := chi.URLParam(r, "service") + //TODO: validate service + if service == "" { + bhs.log.Warn("service is empty") + w.WriteHeader(http.StatusBadRequest) + return + } + + // read request body + body, err := io.ReadAll(r.Body) + if err != nil { + bhs.log.Error("Failed to read request body", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + sc := ServiceCred{} + err = json.Unmarshal(body, &sc) + if err != nil { + bhs.log.Error("Failed to unmarshal request body", "err", err) + w.WriteHeader(http.StatusBadRequest) + return + } + ecdsaBytes, err := hex.DecodeString(sc.EcdsaPubkey) + if err != nil { + bhs.log.Error("Failed to decode ecdsa public key", "err", err) + w.WriteHeader(http.StatusBadRequest) + return + } + err = bhs.builderHubService.RegisterCredentialsForBuilder(r.Context(), builder.Name, service, sc.TlsCert, ecdsaBytes, measurement.Hash, measurement.AttestationType) + if err != nil { + bhs.log.Error("Failed to register credentials", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } diff --git a/ports/types.go b/ports/types.go index 8ca8a72..6d46a82 100644 --- a/ports/types.go +++ b/ports/types.go @@ -62,9 +62,9 @@ func fromDomainBuilderWithServices(builder *domain.BuilderWithServices) BuilderW } type Measurement struct { - Hash string `json:"measurement_hash"` - AttestationType string `json:"attestation_type"` - Measurement map[string]string `json:"measurement"` + Hash string `json:"measurement_hash"` + AttestationType string `json:"attestation_type"` + Measurement map[string]domain.SingleMeasurement `json:"measurement"` } func fromDomainMeasurement(measurement *domain.Measurement) Measurement { diff --git a/ports/types_test.go b/ports/types_test.go deleted file mode 100644 index 9844a60..0000000 --- a/ports/types_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package ports - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestJson(t *testing.T) { - b := BuilderWithServiceCreds{ - Ip: "1.2.3.4", - ServiceCreds: map[string]Service{ - "of-proxy": { - TlsCert: "cert", - EcdsaPubkey: "pubkey", - }, - "rbuilder": { - TlsCert: "cert", - EcdsaPubkey: "pubkey", - }, - }, - } - bts, err := json.Marshal(b) - require.NoError(t, err) - require.Equal(t, string(bts), `{"ip":"1.2.3.4","of-proxy":{"tls_cert":"cert","ecdsa_pubkey":"pubkey"},"rbuilder":{"tls_cert":"cert","ecdsa_pubkey":"pubkey"}}`) -} diff --git a/schema/000_init.sql b/schema/000_init.sql index 7d00e7f..bafe415 100644 --- a/schema/000_init.sql +++ b/schema/000_init.sql @@ -1,22 +1,24 @@ -CREATE TABLE builders ( - name VARCHAR(255) PRIMARY KEY, - ip_address INET NOT NULL, - is_active BOOLEAN NOT NULL DEFAULT true, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - deprecated_at TIMESTAMP WITH TIME ZONE, - CONSTRAINT active_only_when_not_deprecated CHECK ( - (is_active = true AND deprecated_at IS NULL) OR - (is_active = false) - ) +CREATE TABLE builders +( + name VARCHAR(255) PRIMARY KEY, + ip_address INET NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + deprecated_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT active_only_when_not_deprecated CHECK ( + (is_active = true AND deprecated_at IS NULL) OR + (is_active = false) + ) ); -- Add an index on ip_address for faster lookups -CREATE INDEX idx_builders_ip_address ON builders(ip_address); +CREATE INDEX idx_builders_ip_address ON builders (ip_address); -- Trigger to automatically update the updated_at timestamp CREATE OR REPLACE FUNCTION update_builders_updated_at() - RETURNS TRIGGER AS $$ + RETURNS TRIGGER AS +$$ BEGIN NEW.updated_at = CURRENT_TIMESTAMP; RETURN NEW; @@ -24,37 +26,48 @@ END; $$ LANGUAGE plpgsql; CREATE TRIGGER trigger_update_builders_updated_at - BEFORE UPDATE ON builders + BEFORE UPDATE + ON builders FOR EACH ROW EXECUTE FUNCTION update_builders_updated_at(); -- Measurements Whitelist table -CREATE TABLE measurements_whitelist ( - hash BYTEA PRIMARY KEY , -- hash of the measurement - attestation_type TEXT NOT NULL, -- attestation type of the measurement - measurement JSONB NOT NULL, - is_active BOOLEAN NOT NULL DEFAULT true, - - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - deprecated_at TIMESTAMP WITH TIME ZONE, - CONSTRAINT active_only_when_not_deprecated CHECK ( - (is_active = true AND deprecated_at IS NULL) OR - (is_active = false) - ) +CREATE TABLE measurements_whitelist +( + id SERIAL PRIMARY KEY, -- new serial primary key + hash BYTEA NOT NULL, -- hash of the measurement + attestation_type TEXT NOT NULL, -- attestation type of the measurement + measurement JSONB NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT true, + + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + deprecated_at TIMESTAMP WITH TIME ZONE, + + CONSTRAINT active_only_when_not_deprecated CHECK ( + (is_active = true AND deprecated_at IS NULL) OR + (is_active = false) + ), + + CONSTRAINT unique_hash_attestation_type UNIQUE (hash, attestation_type) ); - -CREATE TABLE service_credential_registrations ( - id SERIAL PRIMARY KEY, - builder_name VARCHAR(255) REFERENCES builders(name), - service TEXT NOT NULL, - tls_cert TEXT, - ecdsa_pubkey BYTEA, - is_active BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +CREATE TABLE service_credential_registrations +( + id SERIAL PRIMARY KEY, + builder_name VARCHAR(255) REFERENCES builders (name), + service TEXT NOT NULL, + tls_cert TEXT, + ecdsa_pubkey BYTEA, + is_active BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + deprecated_at TIMESTAMP WITH TIME ZONE, + measurement_id INT REFERENCES measurements_whitelist (id), + CONSTRAINT active_only_when_not_deprecated CHECK ( + (is_active = true AND deprecated_at IS NULL) OR + (is_active = false) + ) ); CREATE UNIQUE INDEX idx_unique_active_credential_per_builder_service @@ -63,7 +76,8 @@ CREATE UNIQUE INDEX idx_unique_active_credential_per_builder_service -- Trigger to automatically update the updated_at timestamp CREATE OR REPLACE FUNCTION update_service_credential_registrations_updated_at() - RETURNS TRIGGER AS $$ + RETURNS TRIGGER AS +$$ BEGIN NEW.updated_at = CURRENT_TIMESTAMP; RETURN NEW; @@ -71,22 +85,23 @@ END; $$ LANGUAGE plpgsql; CREATE TRIGGER trigger_update_service_credential_registrations_updated_at - BEFORE UPDATE ON service_credential_registrations + BEFORE UPDATE + ON service_credential_registrations FOR EACH ROW EXECUTE FUNCTION update_service_credential_registrations_updated_at(); -- Updated builder_configs table -CREATE TABLE builder_configs ( - id SERIAL PRIMARY KEY, - builder_name VARCHAR(255) REFERENCES builders(name), -- references name - config JSONB NOT NULL, - is_active BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +CREATE TABLE builder_configs +( + id SERIAL PRIMARY KEY, + builder_name VARCHAR(255) REFERENCES builders (name), -- references name + config JSONB NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP ); - -- Add a new constraint to ensure only one active config per builder ALTER TABLE builder_configs ADD CONSTRAINT unique_active_config_per_builder @@ -95,3 +110,11 @@ ALTER TABLE builder_configs +CREATE TABLE event_log +( + id SERIAL PRIMARY KEY, -- Unique identifier for each event + event_name TEXT NOT NULL, -- Name of the event + builder_name VARCHAR(255) REFERENCES builders (name) ON DELETE CASCADE, -- Reference to builder + measurement_id INT REFERENCES measurements_whitelist (id) ON DELETE SET NULL, -- Reference to used measurement + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP -- Timestamp when the event occurred +); From 6fb38a0fd240d718517b00bb791bd62fb000b787 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Fri, 11 Oct 2024 15:07:22 +0200 Subject: [PATCH 4/6] change measurement format --- adapters/database/service.go | 26 +++++++++++----------- adapters/database/types.go | 4 ++-- application/service.go | 42 ++++++++++++++++++++++++++++-------- cmd/httpserver/main.go | 7 +++--- domain/types.go | 21 ++++++------------ ports/http_handler.go | 31 ++++++++++++++++---------- ports/types.go | 4 ++-- schema/000_init.sql | 13 ++--------- 8 files changed, 83 insertions(+), 65 deletions(-) diff --git a/adapters/database/service.go b/adapters/database/service.go index 313121b..102e326 100644 --- a/adapters/database/service.go +++ b/adapters/database/service.go @@ -37,16 +37,18 @@ func (s *Service) Close() error { } // GetMeasurementByTypeAndHash retrieves a measurement by OID and hash -func (s *Service) GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) { - var m Measurement - err := s.DB.Get(&m, ` - SELECT * FROM measurements_whitelist - WHERE attestation_type = $1 AND hash = $2 AND is_active = true - `, attestationType, hash) - if errors.Is(err, sql.ErrNoRows) { - return nil, domain.ErrNotFound +func (s *Service) GetActiveMeasurementsByType(ctx context.Context, attestationType string) ([]domain.Measurement, error) { + var measurements []Measurement + err := s.DB.SelectContext(ctx, &measurements, `SELECT * FROM measurements_whitelist WHERE is_active=true AND attestation_type=$1`, attestationType) + var domainMeasurements []domain.Measurement + for _, m := range measurements { + domainM, err := convertMeasurementToDomain(m) + if err != nil { + return nil, err + } + domainMeasurements = append(domainMeasurements, *domainM) } - return convertMeasurementToDomain(m) + return domainMeasurements, err } // GetBuilderByIP retrieves a builder by IP address @@ -88,7 +90,7 @@ func (s *Service) GetActiveMeasurements(ctx context.Context) ([]domain.Measureme // RegisterCredentialsForBuilder registers new credentials for a builder, deprecating all previous credentials // It uses hash and attestation_type to fetch the corresponding measurement_id via a subquery. -func (s *Service) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error { +func (s *Service) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey []byte, measurementName string, attestationType string) error { // Start a transaction tx, err := s.DB.BeginTx(ctx, nil) if err != nil { @@ -116,9 +118,9 @@ func (s *Service) RegisterCredentialsForBuilder(ctx context.Context, builderName INSERT INTO service_credential_registrations (builder_name, service, tls_cert, ecdsa_pubkey, is_active, measurement_id) VALUES ($1, $2, $3, $4, true, - (SELECT id FROM measurements_whitelist WHERE hash = $5 AND attestation_type = $6) + (SELECT id FROM measurements_whitelist WHERE name = $5 AND attestation_type = $6) ) - `, builderName, service, nullableTLSCert, ecdsaPubKey, measurementHash, attestationType) + `, builderName, service, nullableTLSCert, ecdsaPubKey, measurementName, attestationType) if err != nil { return fmt.Errorf("failed to insert credentials for builder %s: %w", builderName, err) } diff --git a/adapters/database/types.go b/adapters/database/types.go index ca0cd24..29c24fc 100644 --- a/adapters/database/types.go +++ b/adapters/database/types.go @@ -19,7 +19,7 @@ type ActiveBuilder struct { type Measurement struct { ID int `db:"id"` - Hash []byte `db:"hash"` + Name string `db:"name"` AttestationType string `db:"attestation_type"` Measurement json.RawMessage `db:"measurement"` IsActive bool `db:"is_active"` @@ -36,7 +36,7 @@ func convertMeasurementToDomain(measurement Measurement) (*domain.Measurement, e if err != nil { return nil, err } - m.Hash = measurement.Hash + m.Name = measurement.Name return &m, nil } diff --git a/application/service.go b/application/service.go index 829cdd9..311637a 100644 --- a/application/service.go +++ b/application/service.go @@ -12,10 +12,10 @@ import ( type BuilderDataAccessor interface { GetActiveMeasurements(ctx context.Context) ([]domain.Measurement, error) GetActiveBuildersWithServiceCredentials(ctx context.Context) ([]domain.BuilderWithServices, error) - GetMeasurementByTypeAndHash(attestationType string, hash []byte) (*domain.Measurement, error) + GetActiveMeasurementsByType(ctx context.Context, attestationType string) ([]domain.Measurement, error) GetBuilderByIP(ip net.IP) (*domain.Builder, error) GetActiveConfigForBuilder(ctx context.Context, builderName string) (json.RawMessage, error) - RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error + RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey []byte, measurementName string, attestationType string) error } type SecretAccessor interface { @@ -39,8 +39,8 @@ func (b *BuilderHub) GetActiveBuilders(ctx context.Context) ([]domain.BuilderWit return b.dataAccessor.GetActiveBuildersWithServiceCredentials(ctx) } -func (b *BuilderHub) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error { - return b.dataAccessor.RegisterCredentialsForBuilder(ctx, builderName, service, tlsCert, ecdsaPubKey, measurementHash, attestationType) +func (b *BuilderHub) RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey []byte, measurementName string, attestationType string) error { + return b.dataAccessor.RegisterCredentialsForBuilder(ctx, builderName, service, tlsCert, ecdsaPubKey, measurementName, attestationType) } func (b *BuilderHub) GetConfigWithSecrets(ctx context.Context, builderName string) ([]byte, error) { @@ -59,15 +59,39 @@ func (b *BuilderHub) GetConfigWithSecrets(ctx context.Context, builderName strin return res, nil } -func (b *BuilderHub) VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) { - _, err := b.dataAccessor.GetMeasurementByTypeAndHash(measurement.AttestationType, measurement.Hash) +func (b *BuilderHub) VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement map[string]string, attestationType string) (*domain.Builder, string, error) { + measurements, err := b.dataAccessor.GetActiveMeasurementsByType(ctx, attestationType) if err != nil { - return nil, fmt.Errorf("failing to fetch corresponding measurement data %x %w", measurement.Hash, err) + return nil, "", fmt.Errorf("failing to fetch corresponding measurement data %s %w", attestationType, err) } + measurementName, err := validateMeasurement(measurement, measurements) + if err != nil { + return nil, "", fmt.Errorf("failing to validate measurement %w", err) + } + builder, err := b.dataAccessor.GetBuilderByIP(ip) if err != nil { // TODO: might avoid logging ip though it should be ok, at least keep it for development state - return nil, fmt.Errorf("failing to fetch builder by ip %s %w", ip.String(), err) + return nil, "", fmt.Errorf("failing to fetch builder by ip %s %w", ip.String(), err) + } + return builder, measurementName, nil +} + +func validateMeasurement(measurement map[string]string, measurementTemplate []domain.Measurement) (string, error) { + for _, m := range measurementTemplate { + if checkMeasurement(measurement, m) { + return m.Name, nil + } + } + return "", domain.ErrNotFound +} + +// validates that all fields from measurementTemplate are the same in measurement +func checkMeasurement(measurement map[string]string, measurementTemplate domain.Measurement) bool { + for k, v := range measurementTemplate.Measurement { + if val, ok := measurement[k]; !ok || val != v.Expected { + return false + } } - return builder, nil + return true } diff --git a/cmd/httpserver/main.go b/cmd/httpserver/main.go index 57b95c6..513b5d1 100644 --- a/cmd/httpserver/main.go +++ b/cmd/httpserver/main.go @@ -30,9 +30,10 @@ var flags = []cli.Flag{ Usage: "address to serve Prometheus metrics", }, &cli.BoolFlag{ - Name: "log-json", - Value: false, - Usage: "log in JSON format", + Name: "log-json", + Value: false, + Usage: "log in JSON format", + EnvVars: []string{"LOG_JSON"}, }, &cli.BoolFlag{ Name: "log-debug", diff --git a/domain/types.go b/domain/types.go index 01e6f80..e58c2f6 100644 --- a/domain/types.go +++ b/domain/types.go @@ -1,19 +1,18 @@ package domain import ( - "crypto/sha256" - "encoding/json" "fmt" "net" ) var ( - ErrNotFound = fmt.Errorf("not found") - ErrIncorrectBuilder = fmt.Errorf("incorrect builder") + ErrNotFound = fmt.Errorf("not found") + ErrIncorrectBuilder = fmt.Errorf("incorrect builder") + ErrInvalidMeasurement = fmt.Errorf("no such active measurement found") ) type Measurement struct { - Hash []byte + Name string AttestationType string Measurement map[string]SingleMeasurement } @@ -22,19 +21,11 @@ type SingleMeasurement struct { Expected string `json:"expected"` } -// CalculateHash calculates the sha256 hash of the given measurements -// cat measurements.json | jq --sort-keys --compact-output --join-output| sha256sum -func CalculateHash(measurements map[string]SingleMeasurement) []byte { - bts, _ := json.Marshal(measurements) - resp := sha256.Sum256(bts) - return resp[:] -} - -func NewMeasurement(attestationType string, measurements map[string]SingleMeasurement) *Measurement { +func NewMeasurement(name, attestationType string, measurements map[string]SingleMeasurement) *Measurement { return &Measurement{ AttestationType: attestationType, Measurement: measurements, - Hash: CalculateHash(measurements), + Name: name, } } diff --git a/ports/http_handler.go b/ports/http_handler.go index add174b..02980b4 100644 --- a/ports/http_handler.go +++ b/ports/http_handler.go @@ -18,9 +18,9 @@ import ( type BuilderHubService interface { GetAllowedMeasurements(ctx context.Context) ([]domain.Measurement, error) GetActiveBuilders(ctx context.Context) ([]domain.BuilderWithServices, error) - VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement *domain.Measurement) (*domain.Builder, error) + VerifyIpAndMeasurements(ctx context.Context, ip net.IP, measurement map[string]string, attestationType string) (*domain.Builder, string, error) GetConfigWithSecrets(ctx context.Context, builderName string) ([]byte, error) - RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey, measurementHash []byte, attestationType string) error + RegisterCredentialsForBuilder(ctx context.Context, builderName, service, tlsCert string, ecdsaPubKey []byte, measurementName string, attestationType string) error } type BuilderHubHandler struct { builderHubService BuilderHubService @@ -33,7 +33,7 @@ func NewBuilderHubHandler(builderHubService BuilderHubService, log *httplog.Logg type AuthData struct { AttestationType string - MeasurementData map[string]domain.SingleMeasurement + MeasurementData map[string]string IP net.IP } @@ -43,15 +43,20 @@ func (bhs *BuilderHubHandler) getAuthData(r *http.Request) (*AuthData, error) { return nil, fmt.Errorf("attestation type is empty %w", ErrInvalidAuthData) } measurementHeader := r.Header.Get(MeasurementHeader) - measurementData := make(map[string]domain.SingleMeasurement) + measurementData := make(map[string]string) err := json.Unmarshal([]byte(measurementHeader), &measurementData) if err != nil { return nil, fmt.Errorf("failed to unmarshal measurement header %w", ErrInvalidAuthData) } - ipHeader := r.Header.Get(ForwardedHeader) + ipHeaders := r.Header.Values(ForwardedHeader) + if len(ipHeaders) == 0 { + return nil, fmt.Errorf("ip header is empty %w", ErrInvalidAuthData) + } + bhs.log.Info("ip headers", "headers", ipHeaders) + ipHeader := ipHeaders[len(ipHeaders)-1] ip := net.ParseIP(ipHeader) if ip == nil { - return nil, fmt.Errorf("failed to parse ip %w", ErrInvalidAuthData) + return nil, fmt.Errorf("failed to parse ip %s %w", ipHeader, ErrInvalidAuthData) } return &AuthData{ @@ -98,7 +103,7 @@ func (bhs *BuilderHubHandler) GetActiveBuilders(w http.ResponseWriter, r *http.R w.WriteHeader(http.StatusForbidden) return } - _, err = bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, domain.NewMeasurement(authData.AttestationType, authData.MeasurementData)) + _, _, err = bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, authData.MeasurementData, authData.AttestationType) if errors.Is(err, domain.ErrNotFound) { bhs.log.Warn("invalid auth data", "error", err) w.WriteHeader(http.StatusForbidden) @@ -163,7 +168,7 @@ func (bhs *BuilderHubHandler) GetConfigSecrets(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusForbidden) return } - builder, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, domain.NewMeasurement(authData.AttestationType, authData.MeasurementData)) + builder, _, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, authData.MeasurementData, authData.AttestationType) if errors.Is(err, domain.ErrNotFound) { bhs.log.Warn("invalid auth data", "error", err) w.WriteHeader(http.StatusForbidden) @@ -188,13 +193,17 @@ func (bhs *BuilderHubHandler) RegisterCredentials(w http.ResponseWriter, r *http w.WriteHeader(http.StatusForbidden) return } - measurement := domain.NewMeasurement(authData.AttestationType, authData.MeasurementData) - builder, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, measurement) + builder, measurementName, err := bhs.builderHubService.VerifyIpAndMeasurements(r.Context(), authData.IP, authData.MeasurementData, authData.AttestationType) if errors.Is(err, domain.ErrNotFound) { bhs.log.Warn("invalid auth data", "error", err) w.WriteHeader(http.StatusForbidden) return } + if err != nil { + bhs.log.Error("failed to verify ip and measurements", "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } service := chi.URLParam(r, "service") //TODO: validate service @@ -224,7 +233,7 @@ func (bhs *BuilderHubHandler) RegisterCredentials(w http.ResponseWriter, r *http w.WriteHeader(http.StatusBadRequest) return } - err = bhs.builderHubService.RegisterCredentialsForBuilder(r.Context(), builder.Name, service, sc.TlsCert, ecdsaBytes, measurement.Hash, measurement.AttestationType) + err = bhs.builderHubService.RegisterCredentialsForBuilder(r.Context(), builder.Name, service, sc.TlsCert, ecdsaBytes, measurementName, authData.AttestationType) if err != nil { bhs.log.Error("Failed to register credentials", "err", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/ports/types.go b/ports/types.go index 6d46a82..ac2e294 100644 --- a/ports/types.go +++ b/ports/types.go @@ -62,14 +62,14 @@ func fromDomainBuilderWithServices(builder *domain.BuilderWithServices) BuilderW } type Measurement struct { - Hash string `json:"measurement_hash"` + Name string `json:"measurement_id"` AttestationType string `json:"attestation_type"` Measurement map[string]domain.SingleMeasurement `json:"measurement"` } func fromDomainMeasurement(measurement *domain.Measurement) Measurement { m := Measurement{ - Hash: hex.EncodeToString(measurement.Hash), + Name: measurement.Name, AttestationType: measurement.AttestationType, Measurement: measurement.Measurement, } diff --git a/schema/000_init.sql b/schema/000_init.sql index bafe415..321f816 100644 --- a/schema/000_init.sql +++ b/schema/000_init.sql @@ -35,7 +35,7 @@ EXECUTE FUNCTION update_builders_updated_at(); CREATE TABLE measurements_whitelist ( id SERIAL PRIMARY KEY, -- new serial primary key - hash BYTEA NOT NULL, -- hash of the measurement + name TEXT NOT NULL, -- hash of the measurement attestation_type TEXT NOT NULL, -- attestation type of the measurement measurement JSONB NOT NULL, is_active BOOLEAN NOT NULL DEFAULT true, @@ -49,7 +49,7 @@ CREATE TABLE measurements_whitelist (is_active = false) ), - CONSTRAINT unique_hash_attestation_type UNIQUE (hash, attestation_type) + CONSTRAINT unique_hash_attestation_type UNIQUE (name, attestation_type) ); @@ -74,15 +74,6 @@ CREATE UNIQUE INDEX idx_unique_active_credential_per_builder_service ON service_credential_registrations (builder_name, service) WHERE is_active = true; --- Trigger to automatically update the updated_at timestamp -CREATE OR REPLACE FUNCTION update_service_credential_registrations_updated_at() - RETURNS TRIGGER AS -$$ -BEGIN - NEW.updated_at = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; CREATE TRIGGER trigger_update_service_credential_registrations_updated_at BEFORE UPDATE From 4281f97b129ad77a6f5d848c53cfe0f5818f7311 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Fri, 11 Oct 2024 15:25:50 +0200 Subject: [PATCH 5/6] add hack to properly process X-Forwarded-For header --- ports/http_handler.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ports/http_handler.go b/ports/http_handler.go index 02980b4..2a4f704 100644 --- a/ports/http_handler.go +++ b/ports/http_handler.go @@ -9,6 +9,7 @@ import ( "io" "net" "net/http" + "strings" "github.com/flashbots/builder-hub/domain" "github.com/go-chi/chi/v5" @@ -52,8 +53,12 @@ func (bhs *BuilderHubHandler) getAuthData(r *http.Request) (*AuthData, error) { if len(ipHeaders) == 0 { return nil, fmt.Errorf("ip header is empty %w", ErrInvalidAuthData) } - bhs.log.Info("ip headers", "headers", ipHeaders) + //NOTE: we need this quite awkward logic since header comes not in the canonical format, with space. ipHeader := ipHeaders[len(ipHeaders)-1] + ipHeaders = strings.Split(ipHeader, ",") + ipHeader = ipHeaders[len(ipHeaders)-1] + ipHeader = strings.TrimSpace(ipHeader) + ip := net.ParseIP(ipHeader) if ip == nil { return nil, fmt.Errorf("failed to parse ip %s %w", ipHeader, ErrInvalidAuthData) From 96498f3346c51db242882dc23235b3387ae539a6 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 22 Oct 2024 14:20:38 +0200 Subject: [PATCH 6/6] name change --- adapters/database/service.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/adapters/database/service.go b/adapters/database/service.go index 102e326..0c92fd8 100644 --- a/adapters/database/service.go +++ b/adapters/database/service.go @@ -216,15 +216,15 @@ func (s *Service) GetActiveBuildersWithServiceCredentials(ctx context.Context) ( // LogEvent creates a new log entry in the event_log table. // It uses hash and attestation_type to fetch the corresponding measurement_id via a subquery. -func (s *Service) LogEvent(ctx context.Context, eventName, builderName, hash, attestationType string) error { +func (s *Service) LogEvent(ctx context.Context, eventName, builderName, name, attestationType string) error { // Insert new event log entry with a subquery to fetch the measurement_id _, err := s.DB.ExecContext(ctx, ` INSERT INTO event_log (event_name, builder_name, measurement_id) VALUES ($1, $2, - (SELECT id FROM measurements_whitelist WHERE hash = $3 AND attestation_type = $4) + (SELECT id FROM measurements_whitelist WHERE name = $3 AND attestation_type = $4) ) - `, eventName, builderName, hash, attestationType) + `, eventName, builderName, name, attestationType) if err != nil { return fmt.Errorf("failed to insert event log for builder %s: %w", builderName, err) }