From 927145cde90d5d144caa59ba0304d2d6377b6813 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 27 Dec 2023 09:43:43 -0800 Subject: [PATCH 01/23] Add Endpoint for fetching DeRegisteredOperators Last 14 days --- disperser/dataapi/operatorsInfo_handlers.go | 23 ++++++++++++ disperser/dataapi/server.go | 41 +++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 disperser/dataapi/operatorsInfo_handlers.go diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go new file mode 100644 index 0000000000..94b1bc987e --- /dev/null +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -0,0 +1,23 @@ +package dataapi + +import ( + "context" +) + +func (s *server) getDeregisterdOperatorsLast14days(ctx context.Context) ([]*DeregisteredOperatorInfo, error) { + + indexedDeregisteredOperatorState, err := s.subgraphClient.QueryIndexedDeregisteredOperatorsInTheLast14Days(ctx) + if err != nil { + return nil, err + } + + // Convert the map to a slice. + operators := indexedDeregisteredOperatorState.Operators + deRegisteredOperators := make([]*DeregisteredOperatorInfo, 0, len(operators)) + + for _, operatorInfo := range operators { + deRegisteredOperators = append(deRegisteredOperators, operatorInfo) + } + + return deRegisteredOperators, nil +} diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index ac4f109add..336dd35eca 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -69,6 +69,11 @@ type ( Data []*BlobMetadataResponse `json:"data"` } + DeRegisteredOperatorsResponse struct { + Meta Meta `json:"meta"` + Data []*DeregisteredOperatorInfo `json:"data"` + } + ErrorResponse struct { Error string `json:"error"` } @@ -130,6 +135,10 @@ func (s *server) Start() error { feed.GET("/blobs", s.FetchBlobsHandler) feed.GET("/blobs/:blob_key", s.FetchBlobHandler) } + operatorsInfo := v1.Group("/operatorsInfo") + { + operatorsInfo.GET("/deRegisteredLast14days", s.FetchBlobsHandler) + } metrics := v1.Group("/metrics") { metrics.GET("/", s.FetchMetricsHandler) @@ -358,6 +367,38 @@ func (s *server) FetchNonSigners(c *gin.Context) { c.JSON(http.StatusOK, metric) } +// FetchDeRegisteredOperators godoc +// +// @Summary Fetch list of DeregisteredOperators Last 14 days +// @Tags OperatorsInfo +// @Produce json +// @Success 200 {object} BlobsResponse +// @Failure 400 {object} ErrorResponse "error: Bad request" +// @Failure 404 {object} ErrorResponse "error: Not found" +// @Failure 500 {object} ErrorResponse "error: Server error" +// @Router /operatorsInfo/deRegisteredLast14days [get] +func (s *server) FetchDeRegisteredOperators(c *gin.Context) { + timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { + s.metrics.ObserveLatency("FetchDeRegisteredOperatorsLast14days", f*1000) // make milliseconds + })) + defer timer.ObserveDuration() + + metadatas, err := s.getDeregisterdOperatorsLast14days(c.Request.Context()) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchDeRegisteredOperatorsLast14days") + errorResponse(c, err) + return + } + + s.metrics.IncrementSuccessfulRequestNum("FetchDeRegisteredOperatorsLast14days") + c.JSON(http.StatusOK, DeRegisteredOperatorsResponse{ + Meta: Meta{ + Size: len(metadatas), + }, + Data: metadatas, + }) +} + func (s *server) getBlobMetadataByBatchesWithLimit(ctx context.Context, limit int) ([]*Batch, []*disperser.BlobMetadata, error) { var ( blobMetadatas = make([]*disperser.BlobMetadata, 0) From fb186078a1a66ff38be59e50285c1a7cabf0642f Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 27 Dec 2023 11:19:04 -0800 Subject: [PATCH 02/23] Update test --- disperser/dataapi/server.go | 2 +- disperser/dataapi/subgraph_client.go | 2 ++ disperser/dataapi/subgraph_client_test.go | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 336dd35eca..dad0232e66 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -137,7 +137,7 @@ func (s *server) Start() error { } operatorsInfo := v1.Group("/operatorsInfo") { - operatorsInfo.GET("/deRegisteredLast14days", s.FetchBlobsHandler) + operatorsInfo.GET("/deRegisteredLast14days", s.FetchDeRegisteredOperators) } metrics := v1.Group("/metrics") { diff --git a/disperser/dataapi/subgraph_client.go b/disperser/dataapi/subgraph_client.go index 427958c948..c775d88770 100644 --- a/disperser/dataapi/subgraph_client.go +++ b/disperser/dataapi/subgraph_client.go @@ -45,6 +45,7 @@ type ( *core.IndexedOperatorInfo // BlockNumber is the block number at which the operator was deregistered. BlockNumber uint + Metadata *Operator } IndexedDeregisteredOperatorState struct { Operators map[core.OperatorID]*DeregisteredOperatorInfo @@ -136,6 +137,7 @@ func (sc *subgraphClient) QueryIndexedDeregisteredOperatorsInTheLast14Days(ctx c operators[operatorId] = &DeregisteredOperatorInfo{ IndexedOperatorInfo: indexedOperatorInfo, BlockNumber: uint(operator.BlockNumber), + Metadata: operator, } } diff --git a/disperser/dataapi/subgraph_client_test.go b/disperser/dataapi/subgraph_client_test.go index 0c0961f89c..81745b0c09 100644 --- a/disperser/dataapi/subgraph_client_test.go +++ b/disperser/dataapi/subgraph_client_test.go @@ -183,6 +183,9 @@ func TestQueryIndexedDeregisteredOperatorsInTheLast14Days(t *testing.T) { assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG2, operator.PubkeyG2) assert.Equal(t, "localhost:32006;32007", operator.Socket) assert.Equal(t, uint64(22), uint64(operator.BlockNumber)) + assert.Equal(t, []byte("0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311"), operator.Metadata.OperatorId) + assert.Equal(t, []byte("0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211"), operator.Metadata.TransactionHash) + assert.Equal(t, uint64(22), uint64(operator.Metadata.BlockNumber)) } func assertGasFees(t *testing.T, gasFees *dataapi.GasFees) { From 0f51d1261654ed1998147034a28fc372de555f6a Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 17 Jan 2024 15:30:56 -0800 Subject: [PATCH 03/23] Check Deregistered Operator is online --- disperser/dataapi/operatorsInfo_handlers.go | 31 +++++++++++++++-- disperser/dataapi/server.go | 37 +++++++++++++-------- disperser/dataapi/subgraph_client.go | 11 +++--- disperser/dataapi/subgraph_client_test.go | 10 +++--- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 94b1bc987e..0e893b6171 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -2,22 +2,47 @@ package dataapi import ( "context" + "fmt" + "net" + "time" + + "github.com/Layr-Labs/eigenda/core" ) -func (s *server) getDeregisterdOperatorsLast14days(ctx context.Context) ([]*DeregisteredOperatorInfo, error) { +func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorInfo, error) { - indexedDeregisteredOperatorState, err := s.subgraphClient.QueryIndexedDeregisteredOperatorsInTheLast14Days(ctx) + indexedDeregisteredOperatorState, err := s.subgraphClient.QueryIndexedDeregisteredOperatorsForTimeWindow(ctx, days) if err != nil { return nil, err } // Convert the map to a slice. + operators := indexedDeregisteredOperatorState.Operators deRegisteredOperators := make([]*DeregisteredOperatorInfo, 0, len(operators)) for _, operatorInfo := range operators { - deRegisteredOperators = append(deRegisteredOperators, operatorInfo) + indexedOperatorInfo := operatorInfo.IndexedOperatorInfo + // Try pinging deregistered if offline add to the list + if !checkIsOperatorOnline(indexedOperatorInfo.Socket) { + fmt.Printf("DataAPI Operator %v is offline\n", indexedOperatorInfo) + deRegisteredOperators = append(deRegisteredOperators, operatorInfo) + } } return deRegisteredOperators, nil } + +// method to check if operator is online +func checkIsOperatorOnline(socket string) bool { + + ipAddress := core.OperatorSocket(socket).GetRetrievalSocket() + timeout := time.Second * 10 + conn, err := net.DialTimeout("tcp", ipAddress, timeout) + if err != nil { + // The server is not responding or not reachable + return false + } + defer conn.Close() // Close the connection after checking + return true +} diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index dad0232e66..efc9cb8b7e 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -69,7 +69,7 @@ type ( Data []*BlobMetadataResponse `json:"data"` } - DeRegisteredOperatorsResponse struct { + DeregisteredOperatorsResponse struct { Meta Meta `json:"meta"` Data []*DeregisteredOperatorInfo `json:"data"` } @@ -137,7 +137,7 @@ func (s *server) Start() error { } operatorsInfo := v1.Group("/operatorsInfo") { - operatorsInfo.GET("/deRegisteredLast14days", s.FetchDeRegisteredOperators) + operatorsInfo.GET("/deRegisteredOperators", s.FetchDeregisteredOperators) } metrics := v1.Group("/metrics") { @@ -367,35 +367,46 @@ func (s *server) FetchNonSigners(c *gin.Context) { c.JSON(http.StatusOK, metric) } -// FetchDeRegisteredOperators godoc +// FetchDeregisteredOperators godoc // -// @Summary Fetch list of DeregisteredOperators Last 14 days +// @Summary Fetch list of DeregisteredOperators for days // @Tags OperatorsInfo // @Produce json // @Success 200 {object} BlobsResponse // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" -// @Router /operatorsInfo/deRegisteredLast14days [get] -func (s *server) FetchDeRegisteredOperators(c *gin.Context) { +// @Router /operatorsInfo/deRegisteredState [get] +func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { - s.metrics.ObserveLatency("FetchDeRegisteredOperatorsLast14days", f*1000) // make milliseconds + s.metrics.ObserveLatency("FetchDeregisteredOperators", f*1000) // make milliseconds })) defer timer.ObserveDuration() - metadatas, err := s.getDeregisterdOperatorsLast14days(c.Request.Context()) + // Get query parameters + // Default Value 14 days + days := c.DefaultQuery("days", "14") // If not specified, defaults to 14 + + // Convert days to integer + daysInt, err := strconv.Atoi(days) if err != nil { - s.metrics.IncrementFailedRequestNum("FetchDeRegisteredOperatorsLast14days") + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid 'days' parameter"}) + return + } + + operatorMetadatas, err := s.getDeregisterdOperatorForDays(c.Request.Context(), int32(daysInt)) + if err != nil { + s.metrics.IncrementFailedRequestNum("FetchDeregisteredOperators") errorResponse(c, err) return } - s.metrics.IncrementSuccessfulRequestNum("FetchDeRegisteredOperatorsLast14days") - c.JSON(http.StatusOK, DeRegisteredOperatorsResponse{ + s.metrics.IncrementSuccessfulRequestNum("FetchDeregisteredOperators") + c.JSON(http.StatusOK, DeregisteredOperatorsResponse{ Meta: Meta{ - Size: len(metadatas), + Size: len(operatorMetadatas), }, - Data: metadatas, + Data: operatorMetadatas, }) } diff --git a/disperser/dataapi/subgraph_client.go b/disperser/dataapi/subgraph_client.go index c775d88770..2bda7beca3 100644 --- a/disperser/dataapi/subgraph_client.go +++ b/disperser/dataapi/subgraph_client.go @@ -16,7 +16,7 @@ type ( QueryBatchesWithLimit(ctx context.Context, limit, skip int) ([]*Batch, error) QueryOperatorsWithLimit(ctx context.Context, limit int) ([]*Operator, error) QueryBatchNonSigningOperatorIdsInInterval(ctx context.Context, intervalSeconds int64) (map[string]int, error) - QueryIndexedDeregisteredOperatorsInTheLast14Days(ctx context.Context) (*IndexedDeregisteredOperatorState, error) + QueryIndexedDeregisteredOperatorsForTimeWindow(ctx context.Context, days int32) (*IndexedDeregisteredOperatorState, error) } Batch struct { Id []byte @@ -42,7 +42,7 @@ type ( TransactionHash []byte } DeregisteredOperatorInfo struct { - *core.IndexedOperatorInfo + IndexedOperatorInfo *core.IndexedOperatorInfo // BlockNumber is the block number at which the operator was deregistered. BlockNumber uint Metadata *Operator @@ -107,9 +107,10 @@ func (sc *subgraphClient) QueryBatchNonSigningOperatorIdsInInterval(ctx context. return batchNonSigningOperatorIds, nil } -func (sc *subgraphClient) QueryIndexedDeregisteredOperatorsInTheLast14Days(ctx context.Context) (*IndexedDeregisteredOperatorState, error) { - last14Days := uint64(time.Now().Add(-_14Days).Unix()) - deregisteredOperators, err := sc.api.QueryDeregisteredOperatorsGreaterThanBlockTimestamp(ctx, last14Days) +func (sc *subgraphClient) QueryIndexedDeregisteredOperatorsForTimeWindow(ctx context.Context, days int32) (*IndexedDeregisteredOperatorState, error) { + // Query all deregistered operators in the last N days. + lastNDays := uint64(time.Now().Add(-time.Duration(days) * 24 * time.Hour).Unix()) + deregisteredOperators, err := sc.api.QueryDeregisteredOperatorsGreaterThanBlockTimestamp(ctx, lastNDays) if err != nil { return nil, err } diff --git a/disperser/dataapi/subgraph_client_test.go b/disperser/dataapi/subgraph_client_test.go index 81745b0c09..3abce5745c 100644 --- a/disperser/dataapi/subgraph_client_test.go +++ b/disperser/dataapi/subgraph_client_test.go @@ -159,12 +159,12 @@ func TestQueryOperators(t *testing.T) { assert.Equal(t, []byte("0x000163fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f212"), operators[1].TransactionHash) } -func TestQueryIndexedDeregisteredOperatorsInTheLast14Days(t *testing.T) { +func TestQueryIndexedDeregisteredOperatorsForTimeWindow(t *testing.T) { mockSubgraphApi := &subgraphmock.MockSubgraphApi{} mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfos, nil) subgraphClient := dataapi.NewSubgraphClient(mockSubgraphApi) - indexedDeregisteredOperatorState, err := subgraphClient.QueryIndexedDeregisteredOperatorsInTheLast14Days(context.Background()) + indexedDeregisteredOperatorState, err := subgraphClient.QueryIndexedDeregisteredOperatorsForTimeWindow(context.Background(), 1) assert.NoError(t, err) operators := indexedDeregisteredOperatorState.Operators @@ -179,9 +179,9 @@ func TestQueryIndexedDeregisteredOperatorsInTheLast14Days(t *testing.T) { expectedIndexedOperatorInfo, err := dataapi.ConvertOperatorInfoGqlToIndexedOperatorInfo(subgraphIndexedOperatorInfos) assert.NoError(t, err) - assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG1, operator.PubkeyG1) - assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG2, operator.PubkeyG2) - assert.Equal(t, "localhost:32006;32007", operator.Socket) + assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG1, operator.IndexedOperatorInfo.PubkeyG1) + assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG2, operator.IndexedOperatorInfo.PubkeyG2) + assert.Equal(t, "localhost:32006;32007", operator.IndexedOperatorInfo.Socket) assert.Equal(t, uint64(22), uint64(operator.BlockNumber)) assert.Equal(t, []byte("0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311"), operator.Metadata.OperatorId) assert.Equal(t, []byte("0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211"), operator.Metadata.TransactionHash) From 064e053463303ce193cc2b227ad9aa9d372a8178 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Fri, 19 Jan 2024 15:10:34 -0800 Subject: [PATCH 04/23] Add Testcases plus parallelize work to check operators are online --- disperser/dataapi/operatorsInfo_handlers.go | 94 ++++- disperser/dataapi/server.go | 11 +- disperser/dataapi/server_test.go | 365 +++++++++++++++++++- disperser/dataapi/subgraph/queries.go | 11 + disperser/dataapi/subgraph_client.go | 3 +- disperser/dataapi/subgraph_client_test.go | 129 ++++++- 6 files changed, 589 insertions(+), 24 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 0e893b6171..48d5cc8d2d 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -2,14 +2,29 @@ package dataapi import ( "context" - "fmt" "net" + "sort" "time" + "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/core" ) -func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorInfo, error) { +type OperatorOnlineStatus struct { + OperatorInfo *Operator + IndexedOperatorInfo *core.IndexedOperatorInfo +} + +var ( + // TODO: this should be configurable + numWorkers = 10 + operatorOnlineStatusChan chan OperatorOnlineStatus + operatorOnlineStatusresultsChan chan *DeRegisteredOperatorMetadata +) + +func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeRegisteredOperatorMetadata, error) { + // Track Time taken to get deregistered operators + startTime := time.Now() indexedDeregisteredOperatorState, err := s.subgraphClient.QueryIndexedDeregisteredOperatorsForTimeWindow(ctx, days) if err != nil { @@ -17,32 +32,79 @@ func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) } // Convert the map to a slice. - operators := indexedDeregisteredOperatorState.Operators - deRegisteredOperators := make([]*DeregisteredOperatorInfo, 0, len(operators)) - for _, operatorInfo := range operators { - indexedOperatorInfo := operatorInfo.IndexedOperatorInfo - // Try pinging deregistered if offline add to the list - if !checkIsOperatorOnline(indexedOperatorInfo.Socket) { - fmt.Printf("DataAPI Operator %v is offline\n", indexedOperatorInfo) - deRegisteredOperators = append(deRegisteredOperators, operatorInfo) - } + operatorOnlineStatusChan = make(chan OperatorOnlineStatus, len(operators)) + operatorOnlineStatusresultsChan = make(chan *DeRegisteredOperatorMetadata, len(operators)) + processOperatorsInParallel(indexedDeregisteredOperatorState, operatorOnlineStatusChan, operatorOnlineStatusresultsChan, s.logger) + + // Collect results of work done + deRegisteredOperatorMetadata := make([]*DeRegisteredOperatorMetadata, 0, len(operators)) + for range operators { + metadata := <-operatorOnlineStatusresultsChan + deRegisteredOperatorMetadata = append(deRegisteredOperatorMetadata, metadata) } - return deRegisteredOperators, nil + // Log the time taken + s.logger.Info("Time taken to get deregistered operators for days: %v", time.Since(startTime)) + sort.Slice(deRegisteredOperatorMetadata, func(i, j int) bool { + return deRegisteredOperatorMetadata[i].BlockNumber < deRegisteredOperatorMetadata[j].BlockNumber + }) + + return deRegisteredOperatorMetadata, nil } // method to check if operator is online -func checkIsOperatorOnline(socket string) bool { - - ipAddress := core.OperatorSocket(socket).GetRetrievalSocket() +func checkIsOperatorOnline(ipAddress string) bool { timeout := time.Second * 10 conn, err := net.DialTimeout("tcp", ipAddress, timeout) if err != nil { - // The server is not responding or not reachable return false } defer conn.Close() // Close the connection after checking return true } + +// Helper Function to Process Operators in Parallel +func processOperatorsInParallel(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeRegisteredOperatorMetadata, logger common.Logger) { + + operators := deRegisteredOperatorState.Operators + // Start worker goroutines + for i := 0; i < numWorkers; i++ { + go func() { + for item := range operatorOnlineStatusChan { + ipAddress := core.OperatorSocket(item.IndexedOperatorInfo.Socket).GetRetrievalSocket() + isOnline := checkIsOperatorOnline(ipAddress) + + // Log the online status + if isOnline { + logger.Debug("Operator %v is online at %s", item.IndexedOperatorInfo, ipAddress) + } else { + logger.Debug("Operator %v is offline at %s", item.IndexedOperatorInfo, ipAddress) + } + + // Create the metadata regardless of online status + metadata := &DeRegisteredOperatorMetadata{ + OperatorId: string(item.OperatorInfo.OperatorId[:]), + BlockNumber: uint(item.OperatorInfo.BlockNumber), + IpAddress: ipAddress, + IsOnline: isOnline, + } + + // Send the metadata to the results channel + operatorOnlineStatusresultsChan <- metadata + } + }() + } + + // Send work to the workers + for _, operatorInfo := range operators { + operatorOnlineStatus := OperatorOnlineStatus{ + OperatorInfo: operatorInfo.Metadata, + IndexedOperatorInfo: operatorInfo.IndexedOperatorInfo, + } + operatorOnlineStatusChan <- operatorOnlineStatus + } + close(operatorOnlineStatusChan) // Close the channel after sending all tasks + +} diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index efc9cb8b7e..822693cae7 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -69,9 +69,16 @@ type ( Data []*BlobMetadataResponse `json:"data"` } + DeRegisteredOperatorMetadata struct { + OperatorId string `json:"operator_id"` + BlockNumber uint `json:"block_number"` + IpAddress string `json:"ip_address"` + IsOnline bool `json:"is_online"` + } + DeregisteredOperatorsResponse struct { - Meta Meta `json:"meta"` - Data []*DeregisteredOperatorInfo `json:"data"` + Meta Meta `json:"meta"` + Data []*DeRegisteredOperatorMetadata `json:"data"` } ErrorResponse struct { diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index f2b0ce9fc3..efb2d0f7e8 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -5,7 +5,9 @@ import ( _ "embed" "encoding/hex" "encoding/json" + "fmt" "io" + "net" "net/http" "net/http/httptest" "testing" @@ -17,6 +19,7 @@ import ( "github.com/Layr-Labs/eigenda/disperser/common/inmem" "github.com/Layr-Labs/eigenda/disperser/dataapi" prommock "github.com/Layr-Labs/eigenda/disperser/dataapi/prometheus/mock" + "github.com/Layr-Labs/eigenda/disperser/dataapi/subgraph" subgraphmock "github.com/Layr-Labs/eigenda/disperser/dataapi/subgraph/mock" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fp" @@ -24,6 +27,7 @@ import ( "github.com/gin-gonic/gin" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "go.uber.org/goleak" ) @@ -58,6 +62,10 @@ var ( gettysburgAddressBytes = []byte("Fourscore and seven years ago our fathers brought forth, on this continent, a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived, and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting-place for those who here gave their lives, that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we cannot dedicate, we cannot consecrate—we cannot hallow—this ground. The brave men, living and dead, who struggled here, have consecrated it far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they here gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom, and that government of the people, by the people, for the people, shall not perish from the earth.") ) +type MockSubgraphClient struct { + mock.Mock +} + func TestFetchBlobHandler(t *testing.T) { r := setUpRouter() @@ -187,7 +195,7 @@ func TestFetchMetricsHandler(t *testing.T) { assert.Equal(t, http.StatusOK, res.StatusCode) assert.Equal(t, 16555.555555555555, response.Throughput) - assert.Equal(t, float64(85.14485344239945), response.CostInGas) + assert.Equal(t, float64(170.2897068847989), response.CostInGas) assert.Equal(t, uint64(1), response.TotalStake) } @@ -231,6 +239,328 @@ func TestFetchMetricsTroughputHandler(t *testing.T) { assert.Equal(t, float64(3.599722666666646e+07), totalThroughput) } +func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorState := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorState[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 1, response.Meta.Size) + assert.Equal(t, 1, len(response.Data)) + + for _, data := range response.Data { + fmt.Printf("Data: %v\n", data) + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", data.OperatorId) + assert.Equal(t, uint(22), data.BlockNumber) + assert.Equal(t, "localhost:32007", data.IpAddress) + } +} + +func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 2, response.Meta.Size) + assert.Equal(t, 2, len(response.Data)) + + operator1Data := response.Data[0] + operator2Data := response.Data[1] + fmt.Printf("Operator1Data: %v\n", operator1Data) + fmt.Printf("Operator2Data: %v\n", operator2Data) + + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) + assert.Equal(t, uint(22), operator1Data.BlockNumber) + assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, false, operator1Data.IsOnline) + + assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) + assert.Equal(t, uint(24), operator2Data.BlockNumber) + assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, false, operator2Data.IsOnline) +} + +func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorState := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorState[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + + // Start the test server + closeServer, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port + if err != nil { + t.Fatalf("Failed to start test server: %v", err) + } + defer closeServer() // Ensure the server is closed after the test + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 1, response.Meta.Size) + assert.Equal(t, 1, len(response.Data)) + assert.Equal(t, true, response.Data[0].IsOnline) +} + +func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOneOnline(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + // Start the test server + closeServer, err := startTestTCPServer("localhost:32009") + if err != nil { + t.Fatalf("Failed to start test server: %v", err) + } + defer closeServer() + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 2, response.Meta.Size) + assert.Equal(t, 2, len(response.Data)) + + operator1Data := response.Data[0] + operator2Data := response.Data[1] + + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) + assert.Equal(t, uint(22), operator1Data.BlockNumber) + assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, false, operator1Data.IsOnline) + + assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) + assert.Equal(t, uint(24), operator2Data.BlockNumber) + assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, true, operator2Data.IsOnline) +} + +func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + // Start the test server + closeServer1, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port + if err != nil { + t.Fatalf("Failed to start test server: %v", err) + } + defer closeServer1() // Ensure the server is closed after the test + + // Start the test server + closeServer2, err := startTestTCPServer("localhost:32009") // Let the OS assign a free port + if err != nil { + t.Fatalf("Failed to start test server: %v", err) + } + defer closeServer2() // Ensure the server is closed after the test + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 2, response.Meta.Size) + assert.Equal(t, 2, len(response.Data)) +} + +func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameBlock(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + indexedOperatorStates[core.OperatorID{2}] = subgraphDeregisteredOperatorInfo3 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphThreeOperatorsDeregistered, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo3, nil).Once() + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 3, response.Meta.Size) + assert.Equal(t, 3, len(response.Data)) + + operator1Data := response.Data[0] + + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) + assert.Equal(t, uint(22), operator1Data.BlockNumber) + assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, false, operator1Data.IsOnline) + + operator2Data := getOperatorData(response.Data, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312") + operator3Data := getOperatorData(response.Data, "0xe24cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568313") + + assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) + assert.Equal(t, uint(24), operator2Data.BlockNumber) + assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, false, operator1Data.IsOnline) + + assert.Equal(t, "0xe24cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568313", operator3Data.OperatorId) + assert.Equal(t, uint(24), operator3Data.BlockNumber) + assert.Equal(t, "localhost:32011", operator3Data.IpAddress) + assert.Equal(t, false, operator3Data.IsOnline) +} + func setUpRouter() *gin.Engine { return gin.Default() } @@ -306,3 +636,36 @@ func makeTestBlob(quorumID core.QuorumID, adversityThreshold uint8) core.Blob { } return blob } + +func startTestTCPServer(address string) (closeFunc func(), err error) { + listener, err := net.Listen("tcp", address) + if err != nil { + return nil, err + } + + closeFunc = func() { listener.Close() } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + conn.Close() // Close the connection immediately + } + }() + + return closeFunc, nil +} + +// Helper to get OperatorData from response +func getOperatorData(operatorMetadtas []*dataapi.DeRegisteredOperatorMetadata, operatorId string) dataapi.DeRegisteredOperatorMetadata { + + for _, operatorMetadata := range operatorMetadtas { + if operatorMetadata.OperatorId == operatorId { + return *operatorMetadata + } + } + return dataapi.DeRegisteredOperatorMetadata{} + +} diff --git a/disperser/dataapi/subgraph/queries.go b/disperser/dataapi/subgraph/queries.go index 266ec4ca6e..d232c35263 100644 --- a/disperser/dataapi/subgraph/queries.go +++ b/disperser/dataapi/subgraph/queries.go @@ -1,6 +1,7 @@ package subgraph import ( + "github.com/Layr-Labs/eigenda/core" "github.com/shurcooL/graphql" ) @@ -47,6 +48,16 @@ type ( // Socket is the socket address of the operator, in the form "host:port" SocketUpdates []SocketUpdates `graphql:"socketUpdates(first: 1, orderBy: blockNumber, orderDirection: desc)"` } + DeregisteredOperatorInfo struct { + IndexedOperatorInfo *IndexedOperatorInfo + // BlockNumber is the block number at which the operator was deregistered. + BlockNumber uint + Metadata *Operator + } + IndexedDeregisteredOperatorState struct { + Operators map[core.OperatorID]*DeregisteredOperatorInfo + } + queryBatches struct { Batches []*Batches `graphql:"batches(orderDirection: $orderDirection, orderBy: $orderBy, first: $first, skip: $skip)"` } diff --git a/disperser/dataapi/subgraph_client.go b/disperser/dataapi/subgraph_client.go index 2bda7beca3..8423ace81d 100644 --- a/disperser/dataapi/subgraph_client.go +++ b/disperser/dataapi/subgraph_client.go @@ -9,8 +9,6 @@ import ( "github.com/Layr-Labs/eigenda/disperser/dataapi/subgraph" ) -const _14Days = 14 * 24 * time.Hour - type ( SubgraphClient interface { QueryBatchesWithLimit(ctx context.Context, limit, skip int) ([]*Batch, error) @@ -210,6 +208,7 @@ func convertOperator(operator *subgraph.Operator) (*Operator, error) { if err != nil { return nil, err } + return &Operator{ Id: []byte(operator.Id), OperatorId: []byte(operator.OperatorId), diff --git a/disperser/dataapi/subgraph_client_test.go b/disperser/dataapi/subgraph_client_test.go index 3abce5745c..ad6ca9c398 100644 --- a/disperser/dataapi/subgraph_client_test.go +++ b/disperser/dataapi/subgraph_client_test.go @@ -42,6 +42,52 @@ var ( }, } + subgraphTwoOperatorsDeregistered = []*subgraph.Operator{ + { + Id: "0x000763fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", + OperatorId: "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", + Operator: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + BlockTimestamp: "1702666046", + BlockNumber: "22", + TransactionHash: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + }, + { + Id: "0x000763bb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f224", + OperatorId: "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", + Operator: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + BlockTimestamp: "1702666070", + BlockNumber: "24", + TransactionHash: "0x000224fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + }, + } + + subgraphThreeOperatorsDeregistered = []*subgraph.Operator{ + { + Id: "0x000763fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", + OperatorId: "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", + Operator: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + BlockTimestamp: "1702666046", + BlockNumber: "22", + TransactionHash: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + }, + { + Id: "0x000763bb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f224", + OperatorId: "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", + Operator: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + BlockTimestamp: "1702666070", + BlockNumber: "24", + TransactionHash: "0x000224fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + }, + { + Id: "0x000763bb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f226", + OperatorId: "0xe24cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568313", + Operator: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f217", + BlockTimestamp: "1702666070", + BlockNumber: "24", + TransactionHash: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f217", + }, + } + subgraphBatches = []*subgraph.Batches{ { Id: "0x000763fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f207", @@ -87,7 +133,7 @@ var ( }, } - subgraphIndexedOperatorInfos = &subgraph.IndexedOperatorInfo{ + subgraphIndexedOperatorInfo1 = &subgraph.IndexedOperatorInfo{ Id: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", PubkeyG1_X: "3336192159512049190945679273141887248666932624338963482128432381981287252980", PubkeyG1_Y: "15195175002875833468883745675063986308012687914999552116603423331534089122704", @@ -105,6 +151,83 @@ var ( }, }, } + + subgraphIndexedOperatorInfo2 = &subgraph.IndexedOperatorInfo{ + Id: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", + PubkeyG1_X: "3336192159512049190945679273141887248666932624338963482128432381981287252980", + PubkeyG1_Y: "15195175002875833468883745675063986308012687914999552116603423331534089122704", + PubkeyG2_X: []graphql.String{ + "21597023645215426396093421944506635812143308313031252511177204078669540440732", + "11405255666568400552575831267661419473985517916677491029848981743882451844775", + }, + PubkeyG2_Y: []graphql.String{ + "9416989242565286095121881312760798075882411191579108217086927390793923664442", + "13612061731370453436662267863740141021994163834412349567410746669651828926551", + }, + SocketUpdates: []subgraph.SocketUpdates{ + { + Socket: "localhost:32008;32009", + }, + }, + } + + subgraphIndexedOperatorInfo3 = &subgraph.IndexedOperatorInfo{ + Id: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", + PubkeyG1_X: "3336192159512049190945679273141887248666932624338963482128432381981287252980", + PubkeyG1_Y: "15195175002875833468883745675063986308012687914999552116603423331534089122704", + PubkeyG2_X: []graphql.String{ + "21597023645215426396093421944506635812143308313031252511177204078669540440732", + "11405255666568400552575831267661419473985517916677491029848981743882451844775", + }, + PubkeyG2_Y: []graphql.String{ + "9416989242565286095121881312760798075882411191579108217086927390793923664442", + "13612061731370453436662267863740141021994163834412349567410746669651828926551", + }, + SocketUpdates: []subgraph.SocketUpdates{ + { + Socket: "localhost:32010;32011", + }, + }, + } + + subgraphDeregisteredOperatorInfo = &subgraph.DeregisteredOperatorInfo{ + IndexedOperatorInfo: subgraphIndexedOperatorInfo1, + BlockNumber: 22, + Metadata: &subgraph.Operator{ + Id: "0x000763fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f222", + OperatorId: "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", + Operator: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + BlockTimestamp: "1702666046", + BlockNumber: "22", + TransactionHash: "0x000223fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f211", + }, + } + + subgraphDeregisteredOperatorInfo2 = &subgraph.DeregisteredOperatorInfo{ + IndexedOperatorInfo: subgraphIndexedOperatorInfo2, + BlockNumber: 24, + Metadata: &subgraph.Operator{ + Id: "0x000763bb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f224", + OperatorId: "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", + Operator: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + BlockTimestamp: "1702666070", + BlockNumber: "24", + TransactionHash: "0x000224fb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f213", + }, + } + + subgraphDeregisteredOperatorInfo3 = &subgraph.DeregisteredOperatorInfo{ + IndexedOperatorInfo: subgraphIndexedOperatorInfo2, + BlockNumber: 24, + Metadata: &subgraph.Operator{ + Id: "0x000763bb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f226", + OperatorId: "0xe24cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568313", + Operator: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f217", + BlockTimestamp: "1702666070", + BlockNumber: "24", + TransactionHash: "0x000224cb86a79eda47c891d8826474d80b6a935ad2a2b5de921933e05c67f320f217", + }, + } ) func TestQueryBatchesWithLimit(t *testing.T) { @@ -162,7 +285,7 @@ func TestQueryOperators(t *testing.T) { func TestQueryIndexedDeregisteredOperatorsForTimeWindow(t *testing.T) { mockSubgraphApi := &subgraphmock.MockSubgraphApi{} mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) - mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfos, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) subgraphClient := dataapi.NewSubgraphClient(mockSubgraphApi) indexedDeregisteredOperatorState, err := subgraphClient.QueryIndexedDeregisteredOperatorsForTimeWindow(context.Background(), 1) assert.NoError(t, err) @@ -176,7 +299,7 @@ func TestQueryIndexedDeregisteredOperatorsForTimeWindow(t *testing.T) { assert.NotNil(t, operator) - expectedIndexedOperatorInfo, err := dataapi.ConvertOperatorInfoGqlToIndexedOperatorInfo(subgraphIndexedOperatorInfos) + expectedIndexedOperatorInfo, err := dataapi.ConvertOperatorInfoGqlToIndexedOperatorInfo(subgraphIndexedOperatorInfo1) assert.NoError(t, err) assert.Equal(t, expectedIndexedOperatorInfo.PubkeyG1, operator.IndexedOperatorInfo.PubkeyG1) From 5d2e2c8d407acf0f682a2078586ca18502392abe Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Fri, 19 Jan 2024 15:30:37 -0800 Subject: [PATCH 05/23] Add Testcase without query param --- disperser/dataapi/server_test.go | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index b92d6a0e19..2b41a77f81 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -345,6 +345,61 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { } } +func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineWithoutDaysQueryParam(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + var response dataapi.DeregisteredOperatorsResponse + err = json.Unmarshal(data, &response) + assert.NoError(t, err) + assert.NotNil(t, response) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, 2, response.Meta.Size) + assert.Equal(t, 2, len(response.Data)) + + operator1Data := response.Data[0] + operator2Data := response.Data[1] + fmt.Printf("Operator1Data: %v\n", operator1Data) + fmt.Printf("Operator2Data: %v\n", operator2Data) + + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) + assert.Equal(t, uint(22), operator1Data.BlockNumber) + assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, false, operator1Data.IsOnline) + + assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) + assert.Equal(t, uint(24), operator2Data.BlockNumber) + assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, false, operator2Data.IsOnline) +} + func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *testing.T) { defer goleak.VerifyNone(t) From 0610dc3c3b826a1efb8b7205c482357a6f970de0 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Fri, 19 Jan 2024 17:10:32 -0800 Subject: [PATCH 06/23] Add comments --- disperser/dataapi/server_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 2b41a77f81..464454c459 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -357,6 +357,8 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineWitho mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + + // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) @@ -412,6 +414,8 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *t mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + + // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) @@ -471,7 +475,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - // Start the test server + // Start test server for Operator closeServer, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) @@ -513,13 +517,15 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) + + // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - // Start the test server + // Start the test server for Operator 2 closeServer, err := startTestTCPServer("localhost:32009") if err != nil { t.Fatalf("Failed to start test server: %v", err) @@ -579,14 +585,14 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - // Start the test server + // Start test server for Operator 1 closeServer1, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) } defer closeServer1() // Ensure the server is closed after the test - // Start the test server + // Start test server for Operator 2 closeServer2, err := startTestTCPServer("localhost:32009") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) @@ -628,6 +634,8 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphThreeOperatorsDeregistered, nil) + + // Set up the mock calls for the three operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo3, nil).Once() From f32685344c68990226fb64cf0b2ae6ddda5b0fb0 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Fri, 19 Jan 2024 17:18:43 -0800 Subject: [PATCH 07/23] fix pr comment --- disperser/dataapi/operatorsInfo_handlers.go | 20 ++++++++++---------- disperser/dataapi/server.go | 4 ++-- disperser/dataapi/server_test.go | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 48d5cc8d2d..40d5c9e87a 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -19,10 +19,10 @@ var ( // TODO: this should be configurable numWorkers = 10 operatorOnlineStatusChan chan OperatorOnlineStatus - operatorOnlineStatusresultsChan chan *DeRegisteredOperatorMetadata + operatorOnlineStatusresultsChan chan *DeregisteredOperatorMetadata ) -func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeRegisteredOperatorMetadata, error) { +func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorMetadata, error) { // Track Time taken to get deregistered operators startTime := time.Now() @@ -35,23 +35,23 @@ func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) operators := indexedDeregisteredOperatorState.Operators operatorOnlineStatusChan = make(chan OperatorOnlineStatus, len(operators)) - operatorOnlineStatusresultsChan = make(chan *DeRegisteredOperatorMetadata, len(operators)) + operatorOnlineStatusresultsChan = make(chan *DeregisteredOperatorMetadata, len(operators)) processOperatorsInParallel(indexedDeregisteredOperatorState, operatorOnlineStatusChan, operatorOnlineStatusresultsChan, s.logger) // Collect results of work done - deRegisteredOperatorMetadata := make([]*DeRegisteredOperatorMetadata, 0, len(operators)) + DeregisteredOperatorMetadata := make([]*DeregisteredOperatorMetadata, 0, len(operators)) for range operators { metadata := <-operatorOnlineStatusresultsChan - deRegisteredOperatorMetadata = append(deRegisteredOperatorMetadata, metadata) + DeregisteredOperatorMetadata = append(DeregisteredOperatorMetadata, metadata) } // Log the time taken s.logger.Info("Time taken to get deregistered operators for days: %v", time.Since(startTime)) - sort.Slice(deRegisteredOperatorMetadata, func(i, j int) bool { - return deRegisteredOperatorMetadata[i].BlockNumber < deRegisteredOperatorMetadata[j].BlockNumber + sort.Slice(DeregisteredOperatorMetadata, func(i, j int) bool { + return DeregisteredOperatorMetadata[i].BlockNumber < DeregisteredOperatorMetadata[j].BlockNumber }) - return deRegisteredOperatorMetadata, nil + return DeregisteredOperatorMetadata, nil } // method to check if operator is online @@ -66,7 +66,7 @@ func checkIsOperatorOnline(ipAddress string) bool { } // Helper Function to Process Operators in Parallel -func processOperatorsInParallel(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeRegisteredOperatorMetadata, logger common.Logger) { +func processOperatorsInParallel(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { operators := deRegisteredOperatorState.Operators // Start worker goroutines @@ -84,7 +84,7 @@ func processOperatorsInParallel(deRegisteredOperatorState *IndexedDeregisteredOp } // Create the metadata regardless of online status - metadata := &DeRegisteredOperatorMetadata{ + metadata := &DeregisteredOperatorMetadata{ OperatorId: string(item.OperatorInfo.OperatorId[:]), BlockNumber: uint(item.OperatorInfo.BlockNumber), IpAddress: ipAddress, diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 80d7dbb570..ff1427ab2d 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -80,7 +80,7 @@ type ( Operators map[string]OperatorNonsigningPercentageMetrics `json:"operators"` } - DeRegisteredOperatorMetadata struct { + DeregisteredOperatorMetadata struct { OperatorId string `json:"operator_id"` BlockNumber uint `json:"block_number"` IpAddress string `json:"ip_address"` @@ -89,7 +89,7 @@ type ( DeregisteredOperatorsResponse struct { Meta Meta `json:"meta"` - Data []*DeRegisteredOperatorMetadata `json:"data"` + Data []*DeregisteredOperatorMetadata `json:"data"` } ErrorResponse struct { diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 464454c459..55710f4e23 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -783,13 +783,13 @@ func startTestTCPServer(address string) (closeFunc func(), err error) { } // Helper to get OperatorData from response -func getOperatorData(operatorMetadtas []*dataapi.DeRegisteredOperatorMetadata, operatorId string) dataapi.DeRegisteredOperatorMetadata { +func getOperatorData(operatorMetadtas []*dataapi.DeregisteredOperatorMetadata, operatorId string) dataapi.DeregisteredOperatorMetadata { for _, operatorMetadata := range operatorMetadtas { if operatorMetadata.OperatorId == operatorId { return *operatorMetadata } } - return dataapi.DeRegisteredOperatorMetadata{} + return dataapi.DeregisteredOperatorMetadata{} } From 4644989e9f3be8d9f052b132ba2dd2cb957d6621 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sat, 20 Jan 2024 15:57:58 -0800 Subject: [PATCH 08/23] Reset Mock between tests --- disperser/dataapi/server.go | 7 +- disperser/dataapi/server_test.go | 166 ++++++++++++++++++++++++++++++- 2 files changed, 169 insertions(+), 4 deletions(-) diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index ff1427ab2d..ca389d32a1 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -427,7 +427,7 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" -// @Router /operatorsInfo/deRegisteredState [get] +// @Router /operatorsInfo/deRegisteredOperators [get] func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { s.metrics.ObserveLatency("FetchDeregisteredOperators", f*1000) // make milliseconds @@ -445,6 +445,11 @@ func (s *server) FetchDeregisteredOperators(c *gin.Context) { return } + if daysInt > 30 { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid 'days' parameter. Max value is 30"}) + return + } + operatorMetadatas, err := s.getDeregisterdOperatorForDays(c.Request.Context(), int32(daysInt)) if err != nil { s.metrics.IncrementFailedRequestNum("FetchDeregisteredOperators") diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 55710f4e23..4bea0d91d4 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -198,7 +198,7 @@ func TestFetchMetricsHandler(t *testing.T) { assert.Equal(t, http.StatusOK, res.StatusCode) assert.Equal(t, 16555.555555555555, response.Throughput) - assert.Equal(t, float64(170.2897068847989), response.CostInGas) + assert.Equal(t, float64(85.14485344239945), response.CostInGas) assert.Equal(t, uint64(1), response.TotalStake) } @@ -343,9 +343,59 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { assert.Equal(t, uint(22), data.BlockNumber) assert.Equal(t, "localhost:32007", data.IpAddress) } + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } -func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineWithoutDaysQueryParam(t *testing.T) { +// func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { + +// defer goleak.VerifyNone(t) + +// r := setUpRouter() + +// indexedOperatorState := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) +// indexedOperatorState[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + +// mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) +// mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) +// mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) +// testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + +// mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + +// // Start test server for Operator +// closeServer, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port +// if err != nil { +// t.Fatalf("Failed to start test server: %v", err) +// } +// defer closeServer() // Ensure the server is closed after the test + +// r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + +// w := httptest.NewRecorder() +// req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/", nil) +// r.ServeHTTP(w, req) + +// res := w.Result() +// defer res.Body.Close() + +// data, err := io.ReadAll(res.Body) +// assert.NoError(t, err) + +// var response dataapi.DeregisteredOperatorsResponse +// err = json.Unmarshal(data, &response) +// assert.NoError(t, err) +// assert.NotNil(t, response) + +// assert.Equal(t, http.StatusOK, res.StatusCode) +// assert.Equal(t, 1, response.Meta.Size) +// assert.Equal(t, 1, len(response.Data)) +// assert.Equal(t, true, response.Data[0].IsOnline) +// } + +func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { defer goleak.VerifyNone(t) @@ -400,9 +450,98 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineWitho assert.Equal(t, uint(24), operator2Data.BlockNumber) assert.Equal(t, "localhost:32009", operator2Data.IpAddress) assert.Equal(t, false, operator2Data.IsOnline) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } -func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *testing.T) { +func TestFetchDeregisteredOperatorsHandlerOperatorInvalidDaysQueryParam(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorStates := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorStates[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + indexedOperatorStates[core.OperatorID{1}] = subgraphDeregisteredOperatorInfo2 + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=ten", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + fmt.Printf("Response: %v\n", res) + + assert.Equal(t, http.StatusBadRequest, res.StatusCode) + + // Assert the response body + var responseBody map[string]string + err := json.Unmarshal(w.Body.Bytes(), &responseBody) + if err != nil { + t.Fatalf("Error unmarshaling response body: %v", err) + } + expectedErrorMessage := "Invalid 'days' parameter" + assert.Equal(t, expectedErrorMessage, responseBody["error"]) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil +} + +func TestFetchDeregisteredOperatorsHandlerOperatorQueryDaysGreaterThan30(t *testing.T) { + + defer goleak.VerifyNone(t) + + r := setUpRouter() + + indexedOperatorState := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) + indexedOperatorState[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) + mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + + mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) + + r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=31", nil) + r.ServeHTTP(w, req) + + res := w.Result() + defer res.Body.Close() + fmt.Printf("Response: %v\n", res) + + assert.Equal(t, http.StatusBadRequest, res.StatusCode) + + // Assert the response body + var responseBody map[string]string + err := json.Unmarshal(w.Body.Bytes(), &responseBody) + if err != nil { + t.Fatalf("Error unmarshaling response body: %v", err) + } + expectedErrorMessage := "Invalid 'days' parameter. Max value is 30" + assert.Equal(t, expectedErrorMessage, responseBody["error"]) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil +} + +func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T) { defer goleak.VerifyNone(t) @@ -438,6 +577,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *t err = json.Unmarshal(data, &response) assert.NoError(t, err) assert.NotNil(t, response) + fmt.Printf("Response: %v\n", response) assert.Equal(t, http.StatusOK, res.StatusCode) assert.Equal(t, 2, response.Meta.Size) @@ -457,6 +597,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOffline(t *t assert.Equal(t, uint(24), operator2Data.BlockNumber) assert.Equal(t, "localhost:32009", operator2Data.IpAddress) assert.Equal(t, false, operator2Data.IsOnline) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { @@ -503,6 +647,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { assert.Equal(t, 1, response.Meta.Size) assert.Equal(t, 1, len(response.Data)) assert.Equal(t, true, response.Data[0].IsOnline) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOneOnline(t *testing.T) { @@ -565,6 +713,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn assert.Equal(t, uint(24), operator2Data.BlockNumber) assert.Equal(t, "localhost:32009", operator2Data.IpAddress) assert.Equal(t, true, operator2Data.IsOnline) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t *testing.T) { @@ -619,6 +771,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t assert.Equal(t, http.StatusOK, res.StatusCode) assert.Equal(t, 2, response.Meta.Size) assert.Equal(t, 2, len(response.Data)) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameBlock(t *testing.T) { @@ -683,6 +839,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB assert.Equal(t, uint(24), operator3Data.BlockNumber) assert.Equal(t, "localhost:32011", operator3Data.IpAddress) assert.Equal(t, false, operator3Data.IsOnline) + + // Reset the mock + mockSubgraphApi.ExpectedCalls = nil + mockSubgraphApi.Calls = nil } func setUpRouter() *gin.Engine { From 651d70eb1dc950f07e940bb0e19ef7743acde440 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sun, 21 Jan 2024 12:37:26 -0800 Subject: [PATCH 09/23] remove commented code --- disperser/dataapi/server_test.go | 81 +++++++++----------------------- 1 file changed, 21 insertions(+), 60 deletions(-) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 4bea0d91d4..cb252cc1da 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net" "net/http" "net/http/httptest" @@ -31,6 +32,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.uber.org/goleak" + "google.golang.org/grpc" ) var ( @@ -349,52 +351,6 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { mockSubgraphApi.Calls = nil } -// func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { - -// defer goleak.VerifyNone(t) - -// r := setUpRouter() - -// indexedOperatorState := make(map[core.OperatorID]*subgraph.DeregisteredOperatorInfo) -// indexedOperatorState[core.OperatorID{0}] = subgraphDeregisteredOperatorInfo - -// mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) -// mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) -// mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) -// testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) - -// mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - -// // Start test server for Operator -// closeServer, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port -// if err != nil { -// t.Fatalf("Failed to start test server: %v", err) -// } -// defer closeServer() // Ensure the server is closed after the test - -// r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) - -// w := httptest.NewRecorder() -// req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/", nil) -// r.ServeHTTP(w, req) - -// res := w.Result() -// defer res.Body.Close() - -// data, err := io.ReadAll(res.Body) -// assert.NoError(t, err) - -// var response dataapi.DeregisteredOperatorsResponse -// err = json.Unmarshal(data, &response) -// assert.NoError(t, err) -// assert.NotNil(t, response) - -// assert.Equal(t, http.StatusOK, res.StatusCode) -// assert.Equal(t, 1, response.Meta.Size) -// assert.Equal(t, 1, len(response.Data)) -// assert.Equal(t, true, response.Data[0].IsOnline) -// } - func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { defer goleak.VerifyNone(t) @@ -620,7 +576,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) // Start test server for Operator - closeServer, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port + closeServer, err := startTestGRPCServer("localhost:32007") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) } @@ -674,7 +630,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) // Start the test server for Operator 2 - closeServer, err := startTestTCPServer("localhost:32009") + closeServer, err := startTestGRPCServer("localhost:32009") if err != nil { t.Fatalf("Failed to start test server: %v", err) } @@ -738,14 +694,14 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) // Start test server for Operator 1 - closeServer1, err := startTestTCPServer("localhost:32007") // Let the OS assign a free port + closeServer1, err := startTestGRPCServer("localhost:32007") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) } defer closeServer1() // Ensure the server is closed after the test // Start test server for Operator 2 - closeServer2, err := startTestTCPServer("localhost:32009") // Let the OS assign a free port + closeServer2, err := startTestGRPCServer("localhost:32009") // Let the OS assign a free port if err != nil { t.Fatalf("Failed to start test server: %v", err) } @@ -921,25 +877,30 @@ func makeTestBlob(quorumID core.QuorumID, adversityThreshold uint8) core.Blob { return blob } -func startTestTCPServer(address string) (closeFunc func(), err error) { - listener, err := net.Listen("tcp", address) +// startTestGRPCServer starts a gRPC server on a specified address. +// It returns a function to stop the server. +func startTestGRPCServer(address string) (stopFunc func(), err error) { + lis, err := net.Listen("tcp", address) if err != nil { return nil, err } - closeFunc = func() { listener.Close() } + grpcServer := grpc.NewServer() + // Here, you would normally register your server with the gRPC server, like so: + // pb.RegisterYourServiceServer(grpcServer, &yourServer{}) + + stopFunc = func() { + grpcServer.Stop() + lis.Close() + } go func() { - for { - conn, err := listener.Accept() - if err != nil { - return - } - conn.Close() // Close the connection immediately + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("Failed to serve: %v", err) } }() - return closeFunc, nil + return stopFunc, nil } // Helper to get OperatorData from response From 8f4220812e0172f3186fb1c86be768f011ba4614 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sun, 21 Jan 2024 12:46:57 -0800 Subject: [PATCH 10/23] add comments --- disperser/dataapi/operatorsInfo_handlers.go | 2 ++ disperser/dataapi/server_test.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 40d5c9e87a..3f01a9d89f 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -55,6 +55,8 @@ func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) } // method to check if operator is online +// Note: This method is least intrusive wat to check if operator is online +// AlternateSolution: Should we add an endpt to check if operator is online? func checkIsOperatorOnline(ipAddress string) bool { timeout := time.Second * 10 conn, err := net.DialTimeout("tcp", ipAddress, timeout) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index cb252cc1da..fc9f5c2ae0 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -886,8 +886,6 @@ func startTestGRPCServer(address string) (stopFunc func(), err error) { } grpcServer := grpc.NewServer() - // Here, you would normally register your server with the gRPC server, like so: - // pb.RegisterYourServiceServer(grpcServer, &yourServer{}) stopFunc = func() { grpcServer.Stop() From ffd8f0ad38dc35b4a817d02b95bc1a770b1a18dd Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sun, 21 Jan 2024 19:12:26 -0800 Subject: [PATCH 11/23] Update method name --- disperser/dataapi/operatorsInfo_handlers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 3f01a9d89f..48be9b8378 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -17,7 +17,7 @@ type OperatorOnlineStatus struct { var ( // TODO: this should be configurable - numWorkers = 10 + numWorkers = 25 operatorOnlineStatusChan chan OperatorOnlineStatus operatorOnlineStatusresultsChan chan *DeregisteredOperatorMetadata ) @@ -36,7 +36,7 @@ func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) operatorOnlineStatusChan = make(chan OperatorOnlineStatus, len(operators)) operatorOnlineStatusresultsChan = make(chan *DeregisteredOperatorMetadata, len(operators)) - processOperatorsInParallel(indexedDeregisteredOperatorState, operatorOnlineStatusChan, operatorOnlineStatusresultsChan, s.logger) + processOperatorOnlineCheck(indexedDeregisteredOperatorState, operatorOnlineStatusChan, operatorOnlineStatusresultsChan, s.logger) // Collect results of work done DeregisteredOperatorMetadata := make([]*DeregisteredOperatorMetadata, 0, len(operators)) @@ -68,7 +68,7 @@ func checkIsOperatorOnline(ipAddress string) bool { } // Helper Function to Process Operators in Parallel -func processOperatorsInParallel(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { +func processOperatorOnlineCheck(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { operators := deRegisteredOperatorState.Operators // Start worker goroutines From ea82a905d028aadf9fbe77abc0fd69880e814c18 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Mon, 22 Jan 2024 07:36:44 -0800 Subject: [PATCH 12/23] Update test --- disperser/dataapi/server_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index fc9f5c2ae0..929466b011 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -728,6 +728,19 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t assert.Equal(t, 2, response.Meta.Size) assert.Equal(t, 2, len(response.Data)) + operator1Data := response.Data[0] + operator2Data := response.Data[1] + + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) + assert.Equal(t, uint(22), operator1Data.BlockNumber) + assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, true, operator1Data.IsOnline) + + assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) + assert.Equal(t, uint(24), operator2Data.BlockNumber) + assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, true, operator2Data.IsOnline) + // Reset the mock mockSubgraphApi.ExpectedCalls = nil mockSubgraphApi.Calls = nil From 0edce34218bec57aa87ecc086c6af9e4c66cb069 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 24 Jan 2024 22:24:25 -0800 Subject: [PATCH 13/23] Fix PR comments --- disperser/dataapi/operatorsInfo_handlers.go | 56 ++++++++++----------- disperser/dataapi/server.go | 8 +-- disperser/dataapi/server_test.go | 36 ++++++------- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index 48be9b8378..d5a1a9b4e0 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -22,7 +22,7 @@ var ( operatorOnlineStatusresultsChan chan *DeregisteredOperatorMetadata ) -func (s *server) getDeregisterdOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorMetadata, error) { +func (s *server) getDeregisteredOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorMetadata, error) { // Track Time taken to get deregistered operators startTime := time.Now() @@ -67,36 +67,12 @@ func checkIsOperatorOnline(ipAddress string) bool { return true } -// Helper Function to Process Operators in Parallel -func processOperatorOnlineCheck(deRegisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { +func processOperatorOnlineCheck(deregisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { + operators := deregisteredOperatorState.Operators - operators := deRegisteredOperatorState.Operators // Start worker goroutines for i := 0; i < numWorkers; i++ { - go func() { - for item := range operatorOnlineStatusChan { - ipAddress := core.OperatorSocket(item.IndexedOperatorInfo.Socket).GetRetrievalSocket() - isOnline := checkIsOperatorOnline(ipAddress) - - // Log the online status - if isOnline { - logger.Debug("Operator %v is online at %s", item.IndexedOperatorInfo, ipAddress) - } else { - logger.Debug("Operator %v is offline at %s", item.IndexedOperatorInfo, ipAddress) - } - - // Create the metadata regardless of online status - metadata := &DeregisteredOperatorMetadata{ - OperatorId: string(item.OperatorInfo.OperatorId[:]), - BlockNumber: uint(item.OperatorInfo.BlockNumber), - IpAddress: ipAddress, - IsOnline: isOnline, - } - - // Send the metadata to the results channel - operatorOnlineStatusresultsChan <- metadata - } - }() + go operatorIsOnlineCheckWorker(operatorOnlineStatusChan, operatorOnlineStatusresultsChan, logger) } // Send work to the workers @@ -108,5 +84,29 @@ func processOperatorOnlineCheck(deRegisteredOperatorState *IndexedDeregisteredOp operatorOnlineStatusChan <- operatorOnlineStatus } close(operatorOnlineStatusChan) // Close the channel after sending all tasks +} + +func operatorIsOnlineCheckWorker(operatorOnlineStatusChan <-chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { + for item := range operatorOnlineStatusChan { + ipAddress := core.OperatorSocket(item.IndexedOperatorInfo.Socket).GetRetrievalSocket() + isOnline := checkIsOperatorOnline(ipAddress) + + // Log the online status + if isOnline { + logger.Debug("Operator %v is online at %s", item.IndexedOperatorInfo, ipAddress) + } else { + logger.Debug("Operator %v is offline at %s", item.IndexedOperatorInfo, ipAddress) + } + // Create the metadata regardless of online status + metadata := &DeregisteredOperatorMetadata{ + OperatorId: string(item.OperatorInfo.OperatorId[:]), + BlockNumber: uint(item.OperatorInfo.BlockNumber), + IpAddress: ipAddress, + IsOnline: isOnline, + } + + // Send the metadata to the results channel + operatorOnlineStatusresultsChan <- metadata + } } diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index ca389d32a1..28edc4eb21 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -155,7 +155,7 @@ func (s *server) Start() error { } operatorsInfo := v1.Group("/operatorsInfo") { - operatorsInfo.GET("/deRegisteredOperators", s.FetchDeregisteredOperators) + operatorsInfo.GET("/deregisteredOperators", s.FetchDeregisteredOperators) } metrics := v1.Group("/metrics") { @@ -420,14 +420,14 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // FetchDeregisteredOperators godoc // -// @Summary Fetch list of DeregisteredOperators for days +// @Summary Fetch list of operators that have been deregistered for days // @Tags OperatorsInfo // @Produce json // @Success 200 {object} BlobsResponse // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" -// @Router /operatorsInfo/deRegisteredOperators [get] +// @Router /operatorsInfo/deregisteredOperators [get] func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { s.metrics.ObserveLatency("FetchDeregisteredOperators", f*1000) // make milliseconds @@ -450,7 +450,7 @@ func (s *server) FetchDeregisteredOperators(c *gin.Context) { return } - operatorMetadatas, err := s.getDeregisterdOperatorForDays(c.Request.Context(), int32(daysInt)) + operatorMetadatas, err := s.getDeregisteredOperatorForDays(c.Request.Context(), int32(daysInt)) if err != nil { s.metrics.IncrementFailedRequestNum("FetchDeregisteredOperators") errorResponse(c, err) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 929466b011..b2e41f4d28 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -318,10 +318,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -371,10 +371,10 @@ func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/", nil) r.ServeHTTP(w, req) res := w.Result() @@ -429,10 +429,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorInvalidDaysQueryParam(t *testi mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=ten", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=ten", nil) r.ServeHTTP(w, req) res := w.Result() @@ -471,10 +471,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorQueryDaysGreaterThan30(t *test mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=31", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=31", nil) r.ServeHTTP(w, req) res := w.Result() @@ -517,10 +517,10 @@ func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -582,10 +582,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { } defer closeServer() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -636,10 +636,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn } defer closeServer() - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -707,10 +707,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t } defer closeServer2() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -768,10 +768,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deRegisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deRegisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() From b57880bc4d63ae9072d070ff1daf7cc7fb5a5485 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Thu, 25 Jan 2024 14:04:57 -0800 Subject: [PATCH 14/23] Fix Test --- disperser/dataapi/server_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 4c5005fdc7..c268b7f8e7 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -314,7 +314,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) @@ -367,7 +367,7 @@ func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) @@ -425,7 +425,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorInvalidDaysQueryParam(t *testi mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) @@ -467,7 +467,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorQueryDaysGreaterThan30(t *test mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) @@ -513,7 +513,7 @@ func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) @@ -571,7 +571,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphOperatorDeregistereds, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil) - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) @@ -625,7 +625,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn // Set up the mock calls for the two operators mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) @@ -689,7 +689,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t mockSubgraphApi.On("QueryDeregisteredOperatorsGreaterThanBlockTimestamp").Return(subgraphTwoOperatorsDeregistered, nil) mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) @@ -764,7 +764,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo1, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo2, nil).Once() mockSubgraphApi.On("QueryOperatorInfoByOperatorIdAtBlockNumber").Return(subgraphIndexedOperatorInfo3, nil).Once() - testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics("9001", &commock.Logger{})) + testDataApiServer = dataapi.NewServer(config, blobstore, prometheusClient, dataapi.NewSubgraphClient(mockSubgraphApi, &commock.Logger{}), mockTx, mockChainState, &commock.Logger{}, dataapi.NewMetrics(nil, "9001", mockLogger)) mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) From 7a45970ff22a7768798e5915c38673d88774ab7b Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Fri, 26 Jan 2024 09:22:46 -0800 Subject: [PATCH 15/23] Add Swagger docs --- disperser/dataapi/docs/docs.go | 40 +++++++++++++++++++++++++++-- disperser/dataapi/docs/swagger.json | 37 ++++++++++++++++++++++++++ disperser/dataapi/docs/swagger.yaml | 24 +++++++++++++++++ disperser/dataapi/server.go | 8 +++--- 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index 1d24bb9062..6c89a2a134 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -1,5 +1,4 @@ -// Code generated by swaggo/swag. DO NOT EDIT. - +// Package docs Code generated by swaggo/swag. DO NOT EDIT package docs import "github.com/swaggo/swag" @@ -310,6 +309,43 @@ const docTemplate = `{ } } } + }, + "/operatorsInfo/deregisteredOperators": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "OperatorsInfo" + ], + "summary": "Fetch list of operators that have been deregistered for days", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/dataapi.BlobsResponse" + } + }, + "400": { + "description": "error: Bad request", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "404": { + "description": "error: Not found", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "500": { + "description": "error: Server error", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + } + } + } } }, "definitions": { diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index 671ea03c55..6b74ea1be5 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -305,6 +305,43 @@ } } } + }, + "/operatorsInfo/deregisteredOperators": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "OperatorsInfo" + ], + "summary": "Fetch list of operators that have been deregistered for days", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/dataapi.BlobsResponse" + } + }, + "400": { + "description": "error: Bad request", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "404": { + "description": "error: Not found", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + }, + "500": { + "description": "error: Server error", + "schema": { + "$ref": "#/definitions/dataapi.ErrorResponse" + } + } + } + } } }, "definitions": { diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index 61004d63a6..d81a38a281 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -343,6 +343,30 @@ paths: summary: Fetch throughput time series tags: - Metrics + /operatorsInfo/deregisteredOperators: + get: + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/dataapi.BlobsResponse' + "400": + description: 'error: Bad request' + schema: + $ref: '#/definitions/dataapi.ErrorResponse' + "404": + description: 'error: Not found' + schema: + $ref: '#/definitions/dataapi.ErrorResponse' + "500": + description: 'error: Server error' + schema: + $ref: '#/definitions/dataapi.ErrorResponse' + summary: Fetch list of operators that have been deregistered for days + tags: + - OperatorsInfo schemes: - https - http diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 28edc4eb21..d9f6a372fb 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -423,10 +423,10 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // @Summary Fetch list of operators that have been deregistered for days // @Tags OperatorsInfo // @Produce json -// @Success 200 {object} BlobsResponse -// @Failure 400 {object} ErrorResponse "error: Bad request" -// @Failure 404 {object} ErrorResponse "error: Not found" -// @Failure 500 {object} ErrorResponse "error: Server error" +// @Success 200 {object} BlobsResponse +// @Failure 400 {object} ErrorResponse "error: Bad request" +// @Failure 404 {object} ErrorResponse "error: Not found" +// @Failure 500 {object} ErrorResponse "error: Server error" // @Router /operatorsInfo/deregisteredOperators [get] func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { From 7590b8c86d01b90e4553fa419755c96c3567e2c5 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Mon, 29 Jan 2024 12:04:51 -0800 Subject: [PATCH 16/23] Add metric endpt for DeRegisteredOperator --- disperser/dataapi/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index d9f6a372fb..01577031b2 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -163,6 +163,7 @@ func (s *server) Start() error { metrics.GET("/throughput", s.FetchMetricsTroughputHandler) metrics.GET("/non_signers", s.FetchNonSigners) metrics.GET("/operator_nonsigning_percentage", s.FetchOperatorsNonsigningPercentageHandler) + metrics.GET("/deregisteredOperators", s.FetchDeregisteredOperators) } swagger := v1.Group("/swagger") { From dc98a1217d86e4ccf6b99c407f4d98cac639fe75 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Mon, 29 Jan 2024 13:48:14 -0800 Subject: [PATCH 17/23] Fix PR Comment --- disperser/dataapi/docs/docs.go | 2 +- disperser/dataapi/docs/swagger.json | 2 +- disperser/dataapi/docs/swagger.yaml | 3 ++- disperser/dataapi/server.go | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index 6c89a2a134..4e88e89f80 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -318,7 +318,7 @@ const docTemplate = `{ "tags": [ "OperatorsInfo" ], - "summary": "Fetch list of operators that have been deregistered for days", + "summary": "Fetch list of operators that have been deregistered for days. Days is a query parameter with a default value of 14 and max value of 30.", "responses": { "200": { "description": "OK", diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index 6b74ea1be5..2b8c157374 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -314,7 +314,7 @@ "tags": [ "OperatorsInfo" ], - "summary": "Fetch list of operators that have been deregistered for days", + "summary": "Fetch list of operators that have been deregistered for days. Days is a query parameter with a default value of 14 and max value of 30.", "responses": { "200": { "description": "OK", diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index d81a38a281..6cc772d788 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -364,7 +364,8 @@ paths: description: 'error: Server error' schema: $ref: '#/definitions/dataapi.ErrorResponse' - summary: Fetch list of operators that have been deregistered for days + summary: Fetch list of operators that have been deregistered for days. Days + is a query parameter with a default value of 14 and max value of 30. tags: - OperatorsInfo schemes: diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 01577031b2..24f97b7a2a 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -421,7 +421,7 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // FetchDeregisteredOperators godoc // -// @Summary Fetch list of operators that have been deregistered for days +// @Summary Fetch list of operators that have been deregistered for days. Days is a query parameter with a default value of 14 and max value of 30. // @Tags OperatorsInfo // @Produce json // @Success 200 {object} BlobsResponse From 66544e1b6869dd944965fcb13bdc8640f06e51e3 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 30 Jan 2024 08:50:40 -0800 Subject: [PATCH 18/23] Fix PR comments --- disperser/dataapi/operatorsInfo_handlers.go | 90 ++++++++++----------- disperser/dataapi/server_test.go | 3 + 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index d5a1a9b4e0..eed1904bf0 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -8,6 +8,7 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/core" + "github.com/gammazero/workerpool" ) type OperatorOnlineStatus struct { @@ -16,8 +17,9 @@ type OperatorOnlineStatus struct { } var ( - // TODO: this should be configurable - numWorkers = 25 + // TODO: Poolsize should be configurable + // Observe performance and tune accordingly + poolSize = 50 operatorOnlineStatusChan chan OperatorOnlineStatus operatorOnlineStatusresultsChan chan *DeregisteredOperatorMetadata ) @@ -36,7 +38,7 @@ func (s *server) getDeregisteredOperatorForDays(ctx context.Context, days int32) operatorOnlineStatusChan = make(chan OperatorOnlineStatus, len(operators)) operatorOnlineStatusresultsChan = make(chan *DeregisteredOperatorMetadata, len(operators)) - processOperatorOnlineCheck(indexedDeregisteredOperatorState, operatorOnlineStatusChan, operatorOnlineStatusresultsChan, s.logger) + processOperatorOnlineCheck(indexedDeregisteredOperatorState, operatorOnlineStatusresultsChan, s.logger) // Collect results of work done DeregisteredOperatorMetadata := make([]*DeregisteredOperatorMetadata, 0, len(operators)) @@ -54,59 +56,57 @@ func (s *server) getDeregisteredOperatorForDays(ctx context.Context, days int32) return DeregisteredOperatorMetadata, nil } -// method to check if operator is online -// Note: This method is least intrusive wat to check if operator is online -// AlternateSolution: Should we add an endpt to check if operator is online? -func checkIsOperatorOnline(ipAddress string) bool { - timeout := time.Second * 10 - conn, err := net.DialTimeout("tcp", ipAddress, timeout) - if err != nil { - return false - } - defer conn.Close() // Close the connection after checking - return true -} - -func processOperatorOnlineCheck(deregisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusChan chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { +func processOperatorOnlineCheck(deregisteredOperatorState *IndexedDeregisteredOperatorState, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { operators := deregisteredOperatorState.Operators + wp := workerpool.New(poolSize) - // Start worker goroutines - for i := 0; i < numWorkers; i++ { - go operatorIsOnlineCheckWorker(operatorOnlineStatusChan, operatorOnlineStatusresultsChan, logger) - } - - // Send work to the workers for _, operatorInfo := range operators { - operatorOnlineStatus := OperatorOnlineStatus{ + operatorStatus := OperatorOnlineStatus{ OperatorInfo: operatorInfo.Metadata, IndexedOperatorInfo: operatorInfo.IndexedOperatorInfo, } - operatorOnlineStatusChan <- operatorOnlineStatus + + // Submit each operator status check to the worker pool + wp.Submit(func() { + checkIsOnlineAndProcessOperator(operatorStatus, operatorOnlineStatusresultsChan, logger) + }) } - close(operatorOnlineStatusChan) // Close the channel after sending all tasks + + wp.StopWait() // Wait for all submitted tasks to complete and stop the pool } -func operatorIsOnlineCheckWorker(operatorOnlineStatusChan <-chan OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { - for item := range operatorOnlineStatusChan { - ipAddress := core.OperatorSocket(item.IndexedOperatorInfo.Socket).GetRetrievalSocket() - isOnline := checkIsOperatorOnline(ipAddress) +func checkIsOnlineAndProcessOperator(operatorStatus OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { + ipAddress := core.OperatorSocket(operatorStatus.IndexedOperatorInfo.Socket).GetRetrievalSocket() + isOnline := checkIsOperatorOnline(ipAddress) - // Log the online status - if isOnline { - logger.Debug("Operator %v is online at %s", item.IndexedOperatorInfo, ipAddress) - } else { - logger.Debug("Operator %v is offline at %s", item.IndexedOperatorInfo, ipAddress) - } + // Log the online status + if isOnline { + logger.Debug("Operator %v is online at %s", operatorStatus.IndexedOperatorInfo, ipAddress) + } else { + logger.Debug("Operator %v is offline at %s", operatorStatus.IndexedOperatorInfo, ipAddress) + } - // Create the metadata regardless of online status - metadata := &DeregisteredOperatorMetadata{ - OperatorId: string(item.OperatorInfo.OperatorId[:]), - BlockNumber: uint(item.OperatorInfo.BlockNumber), - IpAddress: ipAddress, - IsOnline: isOnline, - } + // Create the metadata regardless of online status + metadata := &DeregisteredOperatorMetadata{ + OperatorId: string(operatorStatus.OperatorInfo.OperatorId[:]), + BlockNumber: uint(operatorStatus.OperatorInfo.BlockNumber), + IpAddress: ipAddress, + IsOnline: isOnline, + } - // Send the metadata to the results channel - operatorOnlineStatusresultsChan <- metadata + // Send the metadata to the results channel + operatorOnlineStatusresultsChan <- metadata +} + +// method to check if operator is online +// Note: This method is least intrusive wat to check if operator is online +// AlternateSolution: Should we add an endpt to check if operator is online? +func checkIsOperatorOnline(ipAddress string) bool { + timeout := time.Second * 10 + conn, err := net.DialTimeout("tcp", ipAddress, timeout) + if err != nil { + return false } + defer conn.Close() // Close the connection after checking + return true } diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index c268b7f8e7..655a7597b7 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -660,6 +660,9 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn operator1Data := response.Data[0] operator2Data := response.Data[1] + responseJson := string(data) + fmt.Printf("Response: %v\n", responseJson) + assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) assert.Equal(t, "localhost:32007", operator1Data.IpAddress) From 9afc078f70dc27919ddcfdeaea789db49570d136 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 30 Jan 2024 09:12:19 -0800 Subject: [PATCH 19/23] Fix linter error --- disperser/dataapi/operatorsInfo_handlers.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operatorsInfo_handlers.go index eed1904bf0..a82e6060c2 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operatorsInfo_handlers.go @@ -20,7 +20,6 @@ var ( // TODO: Poolsize should be configurable // Observe performance and tune accordingly poolSize = 50 - operatorOnlineStatusChan chan OperatorOnlineStatus operatorOnlineStatusresultsChan chan *DeregisteredOperatorMetadata ) @@ -36,7 +35,6 @@ func (s *server) getDeregisteredOperatorForDays(ctx context.Context, days int32) // Convert the map to a slice. operators := indexedDeregisteredOperatorState.Operators - operatorOnlineStatusChan = make(chan OperatorOnlineStatus, len(operators)) operatorOnlineStatusresultsChan = make(chan *DeregisteredOperatorMetadata, len(operators)) processOperatorOnlineCheck(indexedDeregisteredOperatorState, operatorOnlineStatusresultsChan, s.logger) From 34ee3b8ea53a731d8d2ee842fb45c26e2fae99ab Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 30 Jan 2024 17:37:46 -0800 Subject: [PATCH 20/23] Fix PR comments --- ...handlers.go => operators_info_handlers.go} | 16 ++++++------- disperser/dataapi/server.go | 2 +- disperser/dataapi/server_test.go | 24 +++++++++---------- disperser/dataapi/subgraph_client.go | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) rename disperser/dataapi/{operatorsInfo_handlers.go => operators_info_handlers.go} (89%) diff --git a/disperser/dataapi/operatorsInfo_handlers.go b/disperser/dataapi/operators_info_handlers.go similarity index 89% rename from disperser/dataapi/operatorsInfo_handlers.go rename to disperser/dataapi/operators_info_handlers.go index a82e6060c2..e9ccd6d6cf 100644 --- a/disperser/dataapi/operatorsInfo_handlers.go +++ b/disperser/dataapi/operators_info_handlers.go @@ -24,7 +24,7 @@ var ( ) func (s *server) getDeregisteredOperatorForDays(ctx context.Context, days int32) ([]*DeregisteredOperatorMetadata, error) { - // Track Time taken to get deregistered operators + // Track time taken to get deregistered operators startTime := time.Now() indexedDeregisteredOperatorState, err := s.subgraphClient.QueryIndexedDeregisteredOperatorsForTimeWindow(ctx, days) @@ -74,21 +74,21 @@ func processOperatorOnlineCheck(deregisteredOperatorState *IndexedDeregisteredOp } func checkIsOnlineAndProcessOperator(operatorStatus OperatorOnlineStatus, operatorOnlineStatusresultsChan chan<- *DeregisteredOperatorMetadata, logger common.Logger) { - ipAddress := core.OperatorSocket(operatorStatus.IndexedOperatorInfo.Socket).GetRetrievalSocket() - isOnline := checkIsOperatorOnline(ipAddress) + socket := core.OperatorSocket(operatorStatus.IndexedOperatorInfo.Socket).GetRetrievalSocket() + isOnline := checkIsOperatorOnline(socket) // Log the online status if isOnline { - logger.Debug("Operator %v is online at %s", operatorStatus.IndexedOperatorInfo, ipAddress) + logger.Debug("Operator %v is online at %s", operatorStatus.IndexedOperatorInfo, socket) } else { - logger.Debug("Operator %v is offline at %s", operatorStatus.IndexedOperatorInfo, ipAddress) + logger.Debug("Operator %v is offline at %s", operatorStatus.IndexedOperatorInfo, socket) } // Create the metadata regardless of online status metadata := &DeregisteredOperatorMetadata{ OperatorId: string(operatorStatus.OperatorInfo.OperatorId[:]), BlockNumber: uint(operatorStatus.OperatorInfo.BlockNumber), - IpAddress: ipAddress, + Socket: socket, IsOnline: isOnline, } @@ -99,9 +99,9 @@ func checkIsOnlineAndProcessOperator(operatorStatus OperatorOnlineStatus, operat // method to check if operator is online // Note: This method is least intrusive wat to check if operator is online // AlternateSolution: Should we add an endpt to check if operator is online? -func checkIsOperatorOnline(ipAddress string) bool { +func checkIsOperatorOnline(socket string) bool { timeout := time.Second * 10 - conn, err := net.DialTimeout("tcp", ipAddress, timeout) + conn, err := net.DialTimeout("tcp", socket, timeout) if err != nil { return false } diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 24f97b7a2a..0b79231a26 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -83,7 +83,7 @@ type ( DeregisteredOperatorMetadata struct { OperatorId string `json:"operator_id"` BlockNumber uint `json:"block_number"` - IpAddress string `json:"ip_address"` + Socket string `json:"socket"` IsOnline bool `json:"is_online"` } diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 655a7597b7..3b5f62b5d9 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -343,7 +343,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { fmt.Printf("Data: %v\n", data) assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", data.OperatorId) assert.Equal(t, uint(22), data.BlockNumber) - assert.Equal(t, "localhost:32007", data.IpAddress) + assert.Equal(t, "localhost:32007", data.Socket) } // Reset the mock @@ -399,12 +399,12 @@ func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) - assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, "localhost:32007", operator1Data.Socket) assert.Equal(t, false, operator1Data.IsOnline) assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) assert.Equal(t, uint(24), operator2Data.BlockNumber) - assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, "localhost:32009", operator2Data.Socket) assert.Equal(t, false, operator2Data.IsOnline) // Reset the mock @@ -546,12 +546,12 @@ func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) - assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, "localhost:32007", operator1Data.Socket) assert.Equal(t, false, operator1Data.IsOnline) assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) assert.Equal(t, uint(24), operator2Data.BlockNumber) - assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, "localhost:32009", operator2Data.Socket) assert.Equal(t, false, operator2Data.IsOnline) // Reset the mock @@ -665,12 +665,12 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) - assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, "localhost:32007", operator1Data.Socket) assert.Equal(t, false, operator1Data.IsOnline) assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) assert.Equal(t, uint(24), operator2Data.BlockNumber) - assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, "localhost:32009", operator2Data.Socket) assert.Equal(t, true, operator2Data.IsOnline) // Reset the mock @@ -736,12 +736,12 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) - assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, "localhost:32007", operator1Data.Socket) assert.Equal(t, true, operator1Data.IsOnline) assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) assert.Equal(t, uint(24), operator2Data.BlockNumber) - assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, "localhost:32009", operator2Data.Socket) assert.Equal(t, true, operator2Data.IsOnline) // Reset the mock @@ -796,7 +796,7 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB assert.Equal(t, "0xe22dae12a0074f20b8fc96a0489376db34075e545ef60c4845d264a732568311", operator1Data.OperatorId) assert.Equal(t, uint(22), operator1Data.BlockNumber) - assert.Equal(t, "localhost:32007", operator1Data.IpAddress) + assert.Equal(t, "localhost:32007", operator1Data.Socket) assert.Equal(t, false, operator1Data.IsOnline) operator2Data := getOperatorData(response.Data, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312") @@ -804,12 +804,12 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB assert.Equal(t, "0xe23cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568312", operator2Data.OperatorId) assert.Equal(t, uint(24), operator2Data.BlockNumber) - assert.Equal(t, "localhost:32009", operator2Data.IpAddress) + assert.Equal(t, "localhost:32009", operator2Data.Socket) assert.Equal(t, false, operator1Data.IsOnline) assert.Equal(t, "0xe24cae12a0074f20b8fc96a0489376db34075e545ef60c4845d264b732568313", operator3Data.OperatorId) assert.Equal(t, uint(24), operator3Data.BlockNumber) - assert.Equal(t, "localhost:32011", operator3Data.IpAddress) + assert.Equal(t, "localhost:32011", operator3Data.Socket) assert.Equal(t, false, operator3Data.IsOnline) // Reset the mock diff --git a/disperser/dataapi/subgraph_client.go b/disperser/dataapi/subgraph_client.go index b6f543a001..660c1a9571 100644 --- a/disperser/dataapi/subgraph_client.go +++ b/disperser/dataapi/subgraph_client.go @@ -184,8 +184,8 @@ func (sc *subgraphClient) QueryNumBatchesByOperatorsInThePastBlockTimestamp(ctx func (sc *subgraphClient) QueryIndexedDeregisteredOperatorsForTimeWindow(ctx context.Context, days int32) (*IndexedDeregisteredOperatorState, error) { // Query all deregistered operators in the last N days. - lastNDays := uint64(time.Now().Add(-time.Duration(days) * 24 * time.Hour).Unix()) - deregisteredOperators, err := sc.api.QueryDeregisteredOperatorsGreaterThanBlockTimestamp(ctx, lastNDays) + lastNDayInSeconds := uint64(time.Now().Add(-time.Duration(days) * 24 * time.Hour).Unix()) + deregisteredOperators, err := sc.api.QueryDeregisteredOperatorsGreaterThanBlockTimestamp(ctx, lastNDayInSeconds) if err != nil { return nil, err } From 23922717d3298713d1d3240868800829b9042af3 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 31 Jan 2024 08:31:47 -0800 Subject: [PATCH 21/23] Fix PR comments --- disperser/dataapi/docs/docs.go | 2 +- disperser/dataapi/docs/swagger.json | 2 +- disperser/dataapi/docs/swagger.yaml | 2 +- disperser/dataapi/operators_info_handlers.go | 2 +- disperser/dataapi/server.go | 6 ++-- disperser/dataapi/server_test.go | 36 ++++++++++---------- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index 4e88e89f80..cded7cb067 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -310,7 +310,7 @@ const docTemplate = `{ } } }, - "/operatorsInfo/deregisteredOperators": { + "/operatorsInfo/deregistered_operators": { "get": { "produces": [ "application/json" diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index 2b8c157374..84471c743d 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -306,7 +306,7 @@ } } }, - "/operatorsInfo/deregisteredOperators": { + "/operatorsInfo/deregistered_operators": { "get": { "produces": [ "application/json" diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index 6cc772d788..0ba62a9fca 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -343,7 +343,7 @@ paths: summary: Fetch throughput time series tags: - Metrics - /operatorsInfo/deregisteredOperators: + /operatorsInfo/deregistered_operators: get: produces: - application/json diff --git a/disperser/dataapi/operators_info_handlers.go b/disperser/dataapi/operators_info_handlers.go index e9ccd6d6cf..2f40660331 100644 --- a/disperser/dataapi/operators_info_handlers.go +++ b/disperser/dataapi/operators_info_handlers.go @@ -97,7 +97,7 @@ func checkIsOnlineAndProcessOperator(operatorStatus OperatorOnlineStatus, operat } // method to check if operator is online -// Note: This method is least intrusive wat to check if operator is online +// Note: This method is least intrusive way to check if operator is online // AlternateSolution: Should we add an endpt to check if operator is online? func checkIsOperatorOnline(socket string) bool { timeout := time.Second * 10 diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 0b79231a26..89d8107548 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -155,7 +155,7 @@ func (s *server) Start() error { } operatorsInfo := v1.Group("/operatorsInfo") { - operatorsInfo.GET("/deregisteredOperators", s.FetchDeregisteredOperators) + operatorsInfo.GET("/deregistered_operators", s.FetchDeregisteredOperators) } metrics := v1.Group("/metrics") { @@ -163,7 +163,7 @@ func (s *server) Start() error { metrics.GET("/throughput", s.FetchMetricsTroughputHandler) metrics.GET("/non_signers", s.FetchNonSigners) metrics.GET("/operator_nonsigning_percentage", s.FetchOperatorsNonsigningPercentageHandler) - metrics.GET("/deregisteredOperators", s.FetchDeregisteredOperators) + metrics.GET("/deregistered_operators", s.FetchDeregisteredOperators) } swagger := v1.Group("/swagger") { @@ -428,7 +428,7 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" -// @Router /operatorsInfo/deregisteredOperators [get] +// @Router /operatorsInfo/deregistered_operators [get] func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { s.metrics.ObserveLatency("FetchDeregisteredOperators", f*1000) // make milliseconds diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 3b5f62b5d9..8693d1ddbe 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -318,10 +318,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -371,10 +371,10 @@ func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/", nil) r.ServeHTTP(w, req) res := w.Result() @@ -429,10 +429,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorInvalidDaysQueryParam(t *testi mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=ten", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=ten", nil) r.ServeHTTP(w, req) res := w.Result() @@ -471,10 +471,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorQueryDaysGreaterThan30(t *test mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=31", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=31", nil) r.ServeHTTP(w, req) res := w.Result() @@ -517,10 +517,10 @@ func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -582,10 +582,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { } defer closeServer() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -636,10 +636,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn } defer closeServer() - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -710,10 +710,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t } defer closeServer2() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -771,10 +771,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregisteredOperators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregisteredOperators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() From 4b7db4f48c57739259a5583d0348f7b515637b1f Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 31 Jan 2024 21:31:35 -0800 Subject: [PATCH 22/23] Fix PR Comment --- disperser/dataapi/docs/docs.go | 33 ++++++++++++++++++++++++++++- disperser/dataapi/docs/swagger.json | 33 ++++++++++++++++++++++++++++- disperser/dataapi/docs/swagger.yaml | 22 ++++++++++++++++++- disperser/dataapi/server.go | 2 +- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index cded7cb067..d3d06dee78 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -323,7 +323,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/dataapi.BlobsResponse" + "$ref": "#/definitions/dataapi.DeregisteredOperatorsResponse" } }, "400": { @@ -461,6 +461,37 @@ const docTemplate = `{ } } }, + "dataapi.DeregisteredOperatorMetadata": { + "type": "object", + "properties": { + "block_number": { + "type": "integer" + }, + "is_online": { + "type": "boolean" + }, + "operator_id": { + "type": "string" + }, + "socket": { + "type": "string" + } + } + }, + "dataapi.DeregisteredOperatorsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/dataapi.DeregisteredOperatorMetadata" + } + }, + "meta": { + "$ref": "#/definitions/dataapi.Meta" + } + } + }, "dataapi.ErrorResponse": { "type": "object", "properties": { diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index 84471c743d..aa9681ecfa 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -319,7 +319,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/dataapi.BlobsResponse" + "$ref": "#/definitions/dataapi.DeregisteredOperatorsResponse" } }, "400": { @@ -457,6 +457,37 @@ } } }, + "dataapi.DeregisteredOperatorMetadata": { + "type": "object", + "properties": { + "block_number": { + "type": "integer" + }, + "is_online": { + "type": "boolean" + }, + "operator_id": { + "type": "string" + }, + "socket": { + "type": "string" + } + } + }, + "dataapi.DeregisteredOperatorsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/dataapi.DeregisteredOperatorMetadata" + } + }, + "meta": { + "$ref": "#/definitions/dataapi.Meta" + } + } + }, "dataapi.ErrorResponse": { "type": "object", "properties": { diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index 0ba62a9fca..6c6947619b 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -80,6 +80,26 @@ definitions: meta: $ref: '#/definitions/dataapi.Meta' type: object + dataapi.DeregisteredOperatorMetadata: + properties: + block_number: + type: integer + is_online: + type: boolean + operator_id: + type: string + socket: + type: string + type: object + dataapi.DeregisteredOperatorsResponse: + properties: + data: + items: + $ref: '#/definitions/dataapi.DeregisteredOperatorMetadata' + type: array + meta: + $ref: '#/definitions/dataapi.Meta' + type: object dataapi.ErrorResponse: properties: error: @@ -351,7 +371,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/dataapi.BlobsResponse' + $ref: '#/definitions/dataapi.DeregisteredOperatorsResponse' "400": description: 'error: Bad request' schema: diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index 89d8107548..b4d7c54367 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -424,7 +424,7 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // @Summary Fetch list of operators that have been deregistered for days. Days is a query parameter with a default value of 14 and max value of 30. // @Tags OperatorsInfo // @Produce json -// @Success 200 {object} BlobsResponse +// @Success 200 {object} DeregisteredOperatorsResponse // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" From 54540cedb656a69abdbaaba195548ac4912b9183 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Wed, 7 Feb 2024 09:18:33 -0800 Subject: [PATCH 23/23] change deregistered operator to kebab-case --- disperser/dataapi/docs/docs.go | 2 +- disperser/dataapi/docs/swagger.json | 2 +- disperser/dataapi/docs/swagger.yaml | 2 +- disperser/dataapi/server.go | 6 ++--- disperser/dataapi/server_test.go | 36 ++++++++++++++--------------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/disperser/dataapi/docs/docs.go b/disperser/dataapi/docs/docs.go index d3d06dee78..d002347db6 100644 --- a/disperser/dataapi/docs/docs.go +++ b/disperser/dataapi/docs/docs.go @@ -310,7 +310,7 @@ const docTemplate = `{ } } }, - "/operatorsInfo/deregistered_operators": { + "/operators-info/deregistered-operators": { "get": { "produces": [ "application/json" diff --git a/disperser/dataapi/docs/swagger.json b/disperser/dataapi/docs/swagger.json index aa9681ecfa..7e52498a6e 100644 --- a/disperser/dataapi/docs/swagger.json +++ b/disperser/dataapi/docs/swagger.json @@ -306,7 +306,7 @@ } } }, - "/operatorsInfo/deregistered_operators": { + "/operators-info/deregistered-operators": { "get": { "produces": [ "application/json" diff --git a/disperser/dataapi/docs/swagger.yaml b/disperser/dataapi/docs/swagger.yaml index 6c6947619b..b63be0b361 100644 --- a/disperser/dataapi/docs/swagger.yaml +++ b/disperser/dataapi/docs/swagger.yaml @@ -363,7 +363,7 @@ paths: summary: Fetch throughput time series tags: - Metrics - /operatorsInfo/deregistered_operators: + /operators-info/deregistered-operators: get: produces: - application/json diff --git a/disperser/dataapi/server.go b/disperser/dataapi/server.go index b4d7c54367..e6e7967932 100644 --- a/disperser/dataapi/server.go +++ b/disperser/dataapi/server.go @@ -153,9 +153,9 @@ func (s *server) Start() error { feed.GET("/blobs", s.FetchBlobsHandler) feed.GET("/blobs/:blob_key", s.FetchBlobHandler) } - operatorsInfo := v1.Group("/operatorsInfo") + operatorsInfo := v1.Group("/operators-info") { - operatorsInfo.GET("/deregistered_operators", s.FetchDeregisteredOperators) + operatorsInfo.GET("/deregistered-operators", s.FetchDeregisteredOperators) } metrics := v1.Group("/metrics") { @@ -428,7 +428,7 @@ func (s *server) FetchOperatorsNonsigningPercentageHandler(c *gin.Context) { // @Failure 400 {object} ErrorResponse "error: Bad request" // @Failure 404 {object} ErrorResponse "error: Not found" // @Failure 500 {object} ErrorResponse "error: Server error" -// @Router /operatorsInfo/deregistered_operators [get] +// @Router /operators-info/deregistered-operators [get] func (s *server) FetchDeregisteredOperators(c *gin.Context) { timer := prometheus.NewTimer(prometheus.ObserverFunc(func(f float64) { s.metrics.ObserveLatency("FetchDeregisteredOperators", f*1000) // make milliseconds diff --git a/disperser/dataapi/server_test.go b/disperser/dataapi/server_test.go index 8693d1ddbe..d2f620cc57 100644 --- a/disperser/dataapi/server_test.go +++ b/disperser/dataapi/server_test.go @@ -318,10 +318,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOffline(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -371,10 +371,10 @@ func TestFetchDeregisteredOperatorWithoutDaysQueryParam(t *testing.T) { mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/", nil) r.ServeHTTP(w, req) res := w.Result() @@ -429,10 +429,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorInvalidDaysQueryParam(t *testi mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=ten", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=ten", nil) r.ServeHTTP(w, req) res := w.Result() @@ -471,10 +471,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorQueryDaysGreaterThan30(t *test mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorState, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=31", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=31", nil) r.ServeHTTP(w, req) res := w.Result() @@ -517,10 +517,10 @@ func TestFetchDeregisteredOperatorsHandlerMultiplerOperatorsOffline(t *testing.T mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -582,10 +582,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorOnline(t *testing.T) { } defer closeServer() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -636,10 +636,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOneOfflineOn } defer closeServer() - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -710,10 +710,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsAllOnline(t } defer closeServer2() // Ensure the server is closed after the test - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result() @@ -771,10 +771,10 @@ func TestFetchDeregisteredOperatorsHandlerOperatorMultiplerOperatorsOfflineSameB mockSubgraphApi.On("QueryIndexedDeregisteredOperatorsForTimeWindow").Return(indexedOperatorStates, nil) - r.GET("/v1/operatorsInfo/deregistered_operators/", testDataApiServer.FetchDeregisteredOperators) + r.GET("/v1/operators-info/deregistered-operators/", testDataApiServer.FetchDeregisteredOperators) w := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/v1/operatorsInfo/deregistered_operators/?days=14", nil) + req := httptest.NewRequest(http.MethodGet, "/v1/operators-info/deregistered-operators/?days=14", nil) r.ServeHTTP(w, req) res := w.Result()