From 1e56af65daddb9c61a9bc5e3595715901b20631d Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 18 Sep 2024 17:47:10 -0300 Subject: [PATCH 01/30] remove responded entries after they have been responded or failed to respond --- aggregator/internal/pkg/aggregator.go | 30 +++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 477112f68..f33b890d9 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -61,7 +61,7 @@ type Aggregator struct { // and can start from zero batchesIdxByIdentifierHash map[[32]byte]uint32 - // Stores the taskCreatedBlock for each batch bt batch index + // Stores the taskCreatedBlock for each batch but batch index batchCreatedBlockByIdx map[uint32]uint64 // Stores the TaskResponse for each batch by batchIdentifierHash @@ -282,7 +282,18 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.logger.Info("Aggregator successfully responded to task", "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - + + // If Aggregator successfully responds to task we acquire the task mutex and + // remove task information from the aggregator maps to prevent memory growth. + agg.taskMutex.Lock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + agg.taskMutex.Unlock() + return } @@ -296,6 +307,16 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "merkleRoot", "0x"+hex.EncodeToString(batchData.BatchMerkleRoot[:]), "senderAddress", "0x"+hex.EncodeToString(batchData.SenderAddress[:]), "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) + + // If the aggregator fails to respond to the task we remove the batch information as well. + agg.taskMutex.Lock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + agg.taskMutex.Unlock() } // / Sends response to contract and waits for transaction receipt @@ -365,6 +386,11 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by BatchMerkleRoot: batchMerkleRoot, SenderAddress: senderAddress, } + agg.logger.Info( + "Task Info added in aggregator:", + "Task", batchIndex, + "batchIdentifierHash", batchIdentifierHash, + ) agg.nextBatchIndex += 1 quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} From 2514d255854494002dfc0262e339326d031dacd6 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 18 Sep 2024 17:47:54 -0300 Subject: [PATCH 02/30] comment --- aggregator/internal/pkg/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index f33b890d9..4e68df52d 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -284,7 +284,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // If Aggregator successfully responds to task we acquire the task mutex and - // remove task information from the aggregator maps to prevent memory growth. + // remove task information from the aggregator maps to prevent a memory leak. agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) From 2c92e79eda78026082c62fa5dedc44aada5e21dc Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 23 Sep 2024 16:59:47 -0300 Subject: [PATCH 03/30] move logs outside of mutex --- aggregator/internal/pkg/aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 4e68df52d..6ca931eea 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -285,14 +285,14 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If Aggregator successfully responds to task we acquire the task mutex and // remove task information from the aggregator maps to prevent a memory leak. - agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + agg.taskMutex.Lock() delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") agg.taskMutex.Unlock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") return } From ea592f4d234a78011ef6916a83b135c94506d718 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 23 Sep 2024 17:00:51 -0300 Subject: [PATCH 04/30] move logs in failure to respond --- aggregator/internal/pkg/aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 6ca931eea..66172f630 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -309,14 +309,14 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // If the aggregator fails to respond to the task we remove the batch information as well. - agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + agg.taskMutex.Lock() delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") agg.taskMutex.Unlock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") } // / Sends response to contract and waits for transaction receipt From 3bf3fc0378eac6d4dd3c889ace83982f6ecd155f Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 30 Sep 2024 10:27:09 -0300 Subject: [PATCH 05/30] wrap deletion logic in go func --- aggregator/internal/pkg/aggregator.go | 36 +++++++++++++++------------ 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 66172f630..10a5c7126 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -285,14 +285,16 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If Aggregator successfully responds to task we acquire the task mutex and // remove task information from the aggregator maps to prevent a memory leak. - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - agg.taskMutex.Lock() - delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.taskMutex.Unlock() - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + go func() { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + agg.taskMutex.Lock() + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + agg.taskMutex.Unlock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + }() return } @@ -309,14 +311,16 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // If the aggregator fails to respond to the task we remove the batch information as well. - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - agg.taskMutex.Lock() - delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.taskMutex.Unlock() - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + go func() { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + agg.taskMutex.Lock() + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + agg.taskMutex.Unlock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + }() } // / Sends response to contract and waits for transaction receipt From 5e230859e47e8382e81e496afa0e80a5062a8e19 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 30 Sep 2024 10:54:11 -0300 Subject: [PATCH 06/30] add sleep to go routine --- aggregator/internal/pkg/aggregator.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 10a5c7126..3f3404d8e 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -286,6 +286,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If Aggregator successfully responds to task we acquire the task mutex and // remove task information from the aggregator maps to prevent a memory leak. go func() { + time.Sleep(10 * time.Second) agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") agg.taskMutex.Lock() delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) @@ -312,6 +313,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If the aggregator fails to respond to the task we remove the batch information as well. go func() { + time.Sleep(10 * time.Second) agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") agg.taskMutex.Lock() delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) From a32492c380e31f6b93f9c5f06a7aab4b7d9f252e Mon Sep 17 00:00:00 2001 From: Uriel Mihura <43704209+uri-99@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:34:23 -0300 Subject: [PATCH 07/30] Update aggregator/internal/pkg/aggregator.go --- aggregator/internal/pkg/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 3f3404d8e..6fc076459 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -61,7 +61,7 @@ type Aggregator struct { // and can start from zero batchesIdxByIdentifierHash map[[32]byte]uint32 - // Stores the taskCreatedBlock for each batch but batch index + // Stores the taskCreatedBlock for each batch by batch index batchCreatedBlockByIdx map[uint32]uint64 // Stores the TaskResponse for each batch by batchIdentifierHash From 1c51c9c270a884937585503fd375b0e4abaf2b02 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:10:13 -0300 Subject: [PATCH 08/30] refactor: clearTaskFromMaps function --- aggregator/internal/pkg/aggregator.go | 39 +++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index b817e8552..dd8452e57 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -268,17 +268,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If Aggregator successfully responds to task we acquire the task mutex and // remove task information from the aggregator maps to prevent a memory leak. - go func() { - time.Sleep(10 * time.Second) - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - agg.taskMutex.Lock() - delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.taskMutex.Unlock() - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") - }() + go agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) + return } @@ -294,17 +285,25 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // If the aggregator fails to respond to the task we remove the batch information as well. - go func() { - time.Sleep(10 * time.Second) - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - agg.taskMutex.Lock() - delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchesIdentifierHashByIdx, blsAggServiceResp.TaskIndex) - delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - agg.taskMutex.Unlock() + go agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) +} + +func (agg *Aggregator) clearTaskFromMaps(taskIndex uint32) { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing Task Info from Aggregator in 10 seconds") + time.Sleep(10 * time.Second) + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + + agg.taskMutex.Lock() + defer func() { agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + agg.taskMutex.Unlock() }() + + batchIdentifierHash := agg.batchesIdentifierHashByIdx[taskIndex] + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, taskIndex) + delete(agg.batchesIdentifierHashByIdx, taskIndex) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) } // / Sends response to contract and waits for transaction receipt From 2a031cea4f830e806ed89ab93a5175d5ed20985b Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:16:24 -0300 Subject: [PATCH 09/30] refactor: clearTaskFromMaps is sequential and no longer a goroutine --- aggregator/internal/pkg/aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index dd8452e57..87aedc98c 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -268,7 +268,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA // If Aggregator successfully responds to task we acquire the task mutex and // remove task information from the aggregator maps to prevent a memory leak. - go agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) + agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) return } @@ -285,7 +285,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // If the aggregator fails to respond to the task we remove the batch information as well. - go agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) + agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) } func (agg *Aggregator) clearTaskFromMaps(taskIndex uint32) { From 91fad8ee59a11a307cfbb8df649d7de703474546 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:54:53 -0300 Subject: [PATCH 10/30] feat: refactor now using map to have a garbage collector --- aggregator/internal/pkg/aggregator.go | 90 +++++++++++++++++++-------- 1 file changed, 63 insertions(+), 27 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 87aedc98c..b5dfc8c44 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -12,7 +12,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/yetanotherco/aligned_layer/metrics" - "github.com/Layr-Labs/eigensdk-go/chainio/clients" sdkclients "github.com/Layr-Labs/eigensdk-go/chainio/clients" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/Layr-Labs/eigensdk-go/services/avsregistry" @@ -67,6 +66,9 @@ type Aggregator struct { // Stores the TaskResponse for each batch by batchIdentifierHash batchDataByIdentifierHash map[[32]byte]BatchData + // Stores if a batch has been finalized, either by response or failure to respond + batchIsFinalizedByIdx map[uint32]struct{} // Id in the key list means it is finalized, using empty struct to save memory + // This task index is to communicate with the local BLS // Service. // Note: In case of a reboot it can start from 0 again @@ -106,6 +108,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchesIdxByIdentifierHash := make(map[[32]byte]uint32) batchDataByIdentifierHash := make(map[[32]byte]BatchData) batchCreatedBlockByIdx := make(map[uint32]uint64) + batchIsFinalizedByIdx := make(map[uint32]struct{}) chainioConfig := sdkclients.BuildAllConfig{ EthHttpUrl: aggregatorConfig.BaseConfig.EthRpcUrl, @@ -119,7 +122,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error aggregatorPrivateKey := aggregatorConfig.EcdsaConfig.PrivateKey logger := aggregatorConfig.BaseConfig.Logger - clients, err := clients.BuildAll(chainioConfig, aggregatorPrivateKey, logger) + clients, err := sdkclients.BuildAll(chainioConfig, aggregatorPrivateKey, logger) if err != nil { logger.Errorf("Cannot create sdk clients", "err", err) return nil, err @@ -161,6 +164,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchesIdxByIdentifierHash: batchesIdxByIdentifierHash, batchDataByIdentifierHash: batchDataByIdentifierHash, batchCreatedBlockByIdx: batchCreatedBlockByIdx, + batchIsFinalizedByIdx: batchIsFinalizedByIdx, nextBatchIndex: nextBatchIndex, taskMutex: &sync.Mutex{}, walletMutex: &sync.Mutex{}, @@ -171,6 +175,8 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error metrics: aggregatorMetrics, } + go aggregator.clearTasksFromMaps(garbageCollectorSleep) + return &aggregator, nil } @@ -208,6 +214,10 @@ func (agg *Aggregator) Start(ctx context.Context) error { const MaxSentTxRetries = 5 +const garbageCollectorSleep = 60 * time.Second + +const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second + func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsAggregationServiceResponse) { if blsAggServiceResp.Err != nil { agg.taskMutex.Lock() @@ -266,10 +276,9 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - // If Aggregator successfully responds to task we acquire the task mutex and - // remove task information from the aggregator maps to prevent a memory leak. - agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) - + // Mark the batch as finalized + agg.finalizeBatchIdx(blsAggServiceResp.TaskIndex) + return } @@ -284,26 +293,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "senderAddress", "0x"+hex.EncodeToString(batchData.SenderAddress[:]), "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - // If the aggregator fails to respond to the task we remove the batch information as well. - agg.clearTaskFromMaps(blsAggServiceResp.TaskIndex) -} - -func (agg *Aggregator) clearTaskFromMaps(taskIndex uint32) { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing Task Info from Aggregator in 10 seconds") - time.Sleep(10 * time.Second) - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - - agg.taskMutex.Lock() - defer func() { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") - agg.taskMutex.Unlock() - }() - - batchIdentifierHash := agg.batchesIdentifierHashByIdx[taskIndex] - delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, taskIndex) - delete(agg.batchesIdentifierHashByIdx, taskIndex) - delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + // Aggregator failed to respond to the task, mark the batch as finalized + agg.finalizeBatchIdx(blsAggServiceResp.TaskIndex) } // / Sends response to contract and waits for transaction receipt @@ -383,7 +374,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(QUORUM_THRESHOLD)} - err := agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + err := agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, BLS_AGG_SERVICE_TIMEOUT) // FIXME(marian): When this errors, should we retry initializing new task? Logging fatal for now. if err != nil { agg.logger.Fatalf("BLS aggregation service error when initializing new task: %s", err) @@ -393,3 +384,48 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Adding new task") agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } + +// TODO missing calling this when task expires +func (agg *Aggregator) clearTasksFromMaps(sleep_seconds time.Duration) { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing deprecated Task Infos from Maps every %d seconds", sleep_seconds) + + for { + time.Sleep(sleep_seconds) + + agg.AggregatorConfig.BaseConfig.Logger.Info("Garbage collecting") + + // Reading batchIsFinalizedByIdx map without using a lock because worst case scenario is we miss a newly inserted value + // in which case, the value will be catched in the next iteration + + for idx := range agg.batchIsFinalizedByIdx { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") + agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning up finalized task", "taskIndex", idx) + + // Critical section inside anonymous function to ensure defer works + func() { + agg.taskMutex.Lock() + + defer func() { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") + agg.taskMutex.Unlock() + }() + + batchIdentifierHash := agg.batchesIdentifierHashByIdx[idx] + + delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) + delete(agg.batchCreatedBlockByIdx, idx) + delete(agg.batchesIdentifierHashByIdx, idx) + delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + + delete(agg.batchIsFinalizedByIdx, idx) + }() + } + + } +} + +func (agg *Aggregator) finalizeBatchIdx(idx uint32) { + agg.taskMutex.Lock() + agg.batchIsFinalizedByIdx[idx] = struct{}{} //now the key is present, no need to waste memory on a value + agg.taskMutex.Unlock() +} From da173bddf7c963cf0639ee9cf0f638bb893b9d8a Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:59:14 -0300 Subject: [PATCH 11/30] chore: comments --- aggregator/internal/pkg/aggregator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index b5dfc8c44..85c1629c4 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -223,6 +223,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.taskMutex.Lock() batchIdentifierHash := agg.batchesIdentifierHashByIdx[blsAggServiceResp.TaskIndex] agg.logger.Error("BlsAggregationServiceResponse contains an error", "err", blsAggServiceResp.Err, "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) + // TODO here add to garbage collector + agg.taskMutex.Unlock() return } @@ -424,6 +426,7 @@ func (agg *Aggregator) clearTasksFromMaps(sleep_seconds time.Duration) { } } +//TODO consider using a channel func (agg *Aggregator) finalizeBatchIdx(idx uint32) { agg.taskMutex.Lock() agg.batchIsFinalizedByIdx[idx] = struct{}{} //now the key is present, no need to waste memory on a value From 34841b4fea13eef49232786e2b5cf080501bcb17 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:54:49 -0300 Subject: [PATCH 12/30] chore: cargo.lock --- batcher/Cargo.lock | 184 +++++++++++++++++++++++++++++++++------------ 1 file changed, 135 insertions(+), 49 deletions(-) diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index 5038a0843..aee59bbe7 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -38,6 +38,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aes" version = "0.8.4" @@ -192,7 +198,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -641,7 +647,6 @@ dependencies = [ "fastrand", "hex", "http 0.2.12", - "hyper 0.14.29", "ring 0.17.8", "time", "tokio", @@ -652,9 +657,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -664,15 +669,16 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.3.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c5f920ffd1e0526ec9e70e50bf444db50b204395a0fa7016bbf9e31ea1698f" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -680,6 +686,7 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", + "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -790,9 +797,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -830,9 +837,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c4134cf3adaeacff34d588dbe814200357b0c466d730cf1c0d8054384a2de4" +checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -851,9 +858,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.4" +version = "0.60.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ "aws-smithy-types", "bytes", @@ -862,9 +869,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.9" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -902,9 +909,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df4217d39fe940066174e6238310167bf466bfbebf3be0661e53cacccde6313" +checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -929,9 +936,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f" +checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -972,9 +979,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.8" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] @@ -1058,7 +1065,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", "serde", @@ -1398,6 +1405,7 @@ checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1478,9 +1486,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -1488,9 +1496,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -1864,10 +1872,23 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version 0.4.0", "syn 2.0.77", ] +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "tempfile", + "thiserror", + "zeroize", +] + [[package]] name = "digest" version = "0.9.0" @@ -2713,7 +2734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -3396,6 +3417,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower", "tower-service", "tracing", ] @@ -3881,6 +3903,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -4892,7 +4923,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.22.21", + "toml_edit 0.21.1", ] [[package]] @@ -5215,7 +5246,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-rustls 0.24.1", "tower-service", @@ -5224,7 +5255,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots 0.25.4", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -5256,14 +5287,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.13", + "rustls 0.23.11", "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration", + "system-configuration 0.6.1", "tokio", "tokio-native-tls", "tokio-rustls 0.26.0", @@ -5677,6 +5708,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ + "base64 0.22.1", "rustls-pki-types", ] @@ -5821,9 +5853,9 @@ dependencies = [ [[package]] name = "sdd" -version = "0.2.0" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d" +checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc" [[package]] name = "sec1" @@ -6724,6 +6756,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "sysinfo" @@ -6748,7 +6783,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -6761,6 +6807,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -6775,8 +6831,9 @@ checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7005,7 +7062,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.21", + "toml_edit 0.22.15", ] [[package]] @@ -7030,9 +7087,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ "indexmap 2.2.6", "toml_datetime", @@ -7636,6 +7693,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -7654,6 +7741,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -7803,16 +7899,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "ws_stream_wasm" version = "0.7.4" From 56f9b70498704e4769cbad811e5b41679299e88c Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:52:59 -0300 Subject: [PATCH 13/30] feat: chan usage --- aggregator/internal/pkg/aggregator.go | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 85c1629c4..cb14947f8 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -68,6 +68,7 @@ type Aggregator struct { // Stores if a batch has been finalized, either by response or failure to respond batchIsFinalizedByIdx map[uint32]struct{} // Id in the key list means it is finalized, using empty struct to save memory + batchIsFinalizedChan chan uint32 // This task index is to communicate with the local BLS // Service. @@ -109,6 +110,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchDataByIdentifierHash := make(map[[32]byte]BatchData) batchCreatedBlockByIdx := make(map[uint32]uint64) batchIsFinalizedByIdx := make(map[uint32]struct{}) + batchIsFinalizedChan := make(chan uint32) chainioConfig := sdkclients.BuildAllConfig{ EthHttpUrl: aggregatorConfig.BaseConfig.EthRpcUrl, @@ -165,6 +167,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchDataByIdentifierHash: batchDataByIdentifierHash, batchCreatedBlockByIdx: batchCreatedBlockByIdx, batchIsFinalizedByIdx: batchIsFinalizedByIdx, + batchIsFinalizedChan: batchIsFinalizedChan, nextBatchIndex: nextBatchIndex, taskMutex: &sync.Mutex{}, walletMutex: &sync.Mutex{}, @@ -175,7 +178,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error metrics: aggregatorMetrics, } - go aggregator.clearTasksFromMaps(garbageCollectorSleep) + go aggregator.clearTasksFromMaps(garbageCollectorPeriod) return &aggregator, nil } @@ -208,13 +211,16 @@ func (agg *Aggregator) Start(ctx context.Context) error { "taskIndex", blsAggServiceResp.TaskIndex) go agg.handleBlsAggServiceResponse(blsAggServiceResp) + case taskIdx := <-agg.batchIsFinalizedChan: + agg.logger.Info("Batch is finalized", "taskIndex", taskIdx) + agg.finalizeBatchIdx(taskIdx) } } } const MaxSentTxRetries = 5 -const garbageCollectorSleep = 60 * time.Second +const garbageCollectorPeriod = 60 * time.Second const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second @@ -223,7 +229,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.taskMutex.Lock() batchIdentifierHash := agg.batchesIdentifierHashByIdx[blsAggServiceResp.TaskIndex] agg.logger.Error("BlsAggregationServiceResponse contains an error", "err", blsAggServiceResp.Err, "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - // TODO here add to garbage collector + // Task errored, mark as finalized + agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex agg.taskMutex.Unlock() return @@ -279,7 +286,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // Mark the batch as finalized - agg.finalizeBatchIdx(blsAggServiceResp.TaskIndex) + agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex + return } @@ -296,7 +304,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) // Aggregator failed to respond to the task, mark the batch as finalized - agg.finalizeBatchIdx(blsAggServiceResp.TaskIndex) + agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex } // / Sends response to contract and waits for transaction receipt @@ -387,25 +395,25 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } -// TODO missing calling this when task expires -func (agg *Aggregator) clearTasksFromMaps(sleep_seconds time.Duration) { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing deprecated Task Infos from Maps every %d seconds", sleep_seconds) +// long-lived gorouting that periodically checks and removes finished Tasks from stored Maps +func (agg *Aggregator) clearTasksFromMaps(period time.Duration) { + agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing finalized Task Infos from Maps every %d seconds", period) for { - time.Sleep(sleep_seconds) + time.Sleep(period) - agg.AggregatorConfig.BaseConfig.Logger.Info("Garbage collecting") + agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") // Reading batchIsFinalizedByIdx map without using a lock because worst case scenario is we miss a newly inserted value // in which case, the value will be catched in the next iteration for idx := range agg.batchIsFinalizedByIdx { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning up finalized task", "taskIndex", idx) // Critical section inside anonymous function to ensure defer works func() { agg.taskMutex.Lock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") defer func() { agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") @@ -426,9 +434,7 @@ func (agg *Aggregator) clearTasksFromMaps(sleep_seconds time.Duration) { } } -//TODO consider using a channel +// called in the chan so no need for a mutex func (agg *Aggregator) finalizeBatchIdx(idx uint32) { - agg.taskMutex.Lock() agg.batchIsFinalizedByIdx[idx] = struct{}{} //now the key is present, no need to waste memory on a value - agg.taskMutex.Unlock() } From ce96d6973e822b1f8bd5913c869c2585bfb4a406 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:32:03 -0300 Subject: [PATCH 14/30] chore: cargo --- batcher/Cargo.lock | 44 +++++++------------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index 2ea7469dd..70fc915ae 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -38,12 +38,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - [[package]] name = "aes" version = "0.8.4" @@ -195,7 +189,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1061,7 +1055,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", "serde", @@ -1467,9 +1461,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1477,9 +1471,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -1870,19 +1864,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "tempfile", - "thiserror", - "zeroize", -] - [[package]] name = "digest" version = "0.9.0" @@ -2734,7 +2715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -3348,7 +3329,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -3832,15 +3812,6 @@ dependencies = [ "adler2", ] -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "1.0.2" @@ -5629,7 +5600,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] From 36cb8dfc1081b360ac48a8bfda136134cc7ff8d6 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:41:15 -0300 Subject: [PATCH 15/30] fix: agg retasking expired tasks --- aggregator/internal/pkg/aggregator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index cb14947f8..aec72a409 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -230,7 +230,9 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA batchIdentifierHash := agg.batchesIdentifierHashByIdx[blsAggServiceResp.TaskIndex] agg.logger.Error("BlsAggregationServiceResponse contains an error", "err", blsAggServiceResp.Err, "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) // Task errored, mark as finalized - agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex + // TODO: + // Actually, if I remove this task from the maps, Agg will go and fetch it again from chain + // agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex agg.taskMutex.Unlock() return From 56132e8178c273549e31140121cdb277be69f013 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:24:22 -0300 Subject: [PATCH 16/30] refactor: now delete old tasks --- aggregator/internal/pkg/aggregator.go | 75 ++++++++------------------- core/chainio/avs_reader.go | 47 +++++++++++++++++ 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index aec72a409..3e82b8d7f 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -66,9 +66,6 @@ type Aggregator struct { // Stores the TaskResponse for each batch by batchIdentifierHash batchDataByIdentifierHash map[[32]byte]BatchData - // Stores if a batch has been finalized, either by response or failure to respond - batchIsFinalizedByIdx map[uint32]struct{} // Id in the key list means it is finalized, using empty struct to save memory - batchIsFinalizedChan chan uint32 // This task index is to communicate with the local BLS // Service. @@ -109,8 +106,6 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchesIdxByIdentifierHash := make(map[[32]byte]uint32) batchDataByIdentifierHash := make(map[[32]byte]BatchData) batchCreatedBlockByIdx := make(map[uint32]uint64) - batchIsFinalizedByIdx := make(map[uint32]struct{}) - batchIsFinalizedChan := make(chan uint32) chainioConfig := sdkclients.BuildAllConfig{ EthHttpUrl: aggregatorConfig.BaseConfig.EthRpcUrl, @@ -166,8 +161,6 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error batchesIdxByIdentifierHash: batchesIdxByIdentifierHash, batchDataByIdentifierHash: batchDataByIdentifierHash, batchCreatedBlockByIdx: batchCreatedBlockByIdx, - batchIsFinalizedByIdx: batchIsFinalizedByIdx, - batchIsFinalizedChan: batchIsFinalizedChan, nextBatchIndex: nextBatchIndex, taskMutex: &sync.Mutex{}, walletMutex: &sync.Mutex{}, @@ -178,7 +171,7 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error metrics: aggregatorMetrics, } - go aggregator.clearTasksFromMaps(garbageCollectorPeriod) + go aggregator.clearTasksFromMaps(garbageCollectorPeriod, garbageCollectorTasksAge) return &aggregator, nil } @@ -211,16 +204,14 @@ func (agg *Aggregator) Start(ctx context.Context) error { "taskIndex", blsAggServiceResp.TaskIndex) go agg.handleBlsAggServiceResponse(blsAggServiceResp) - case taskIdx := <-agg.batchIsFinalizedChan: - agg.logger.Info("Batch is finalized", "taskIndex", taskIdx) - agg.finalizeBatchIdx(taskIdx) } } } const MaxSentTxRetries = 5 -const garbageCollectorPeriod = 60 * time.Second +const garbageCollectorPeriod = time.Second * 30 //TODO change to time.Day * 1 +const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second @@ -229,12 +220,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.taskMutex.Lock() batchIdentifierHash := agg.batchesIdentifierHashByIdx[blsAggServiceResp.TaskIndex] agg.logger.Error("BlsAggregationServiceResponse contains an error", "err", blsAggServiceResp.Err, "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - // Task errored, mark as finalized - // TODO: - // Actually, if I remove this task from the maps, Agg will go and fetch it again from chain - // agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex - agg.taskMutex.Unlock() + return } nonSignerPubkeys := []servicemanager.BN254G1Point{} @@ -287,10 +274,6 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - // Mark the batch as finalized - agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex - - return } @@ -305,8 +288,6 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "senderAddress", "0x"+hex.EncodeToString(batchData.SenderAddress[:]), "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - // Aggregator failed to respond to the task, mark the batch as finalized - agg.batchIsFinalizedChan <- blsAggServiceResp.TaskIndex } // / Sends response to contract and waits for transaction receipt @@ -397,46 +378,32 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } -// long-lived gorouting that periodically checks and removes finished Tasks from stored Maps -func (agg *Aggregator) clearTasksFromMaps(period time.Duration) { +// long-lived gorouting that periodically checks and removes old Tasks from stored Maps +func (agg *Aggregator) clearTasksFromMaps(period time.Duration, blocksOld uint64) { agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing finalized Task Infos from Maps every %d seconds", period) + lastIdxDeleted := uint32(0) for { time.Sleep(period) agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") + oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(blocksOld) + if err != nil { + agg.logger.Error("Error getting old task hash, skipping this garbage collect", "err", err) + continue // Retry in the next iteration + } - // Reading batchIsFinalizedByIdx map without using a lock because worst case scenario is we miss a newly inserted value - // in which case, the value will be catched in the next iteration - - for idx := range agg.batchIsFinalizedByIdx { - agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning up finalized task", "taskIndex", idx) - - // Critical section inside anonymous function to ensure defer works - func() { - agg.taskMutex.Lock() - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Removing Task Info from Aggregator") - - defer func() { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Removed Task Info from Aggregator") - agg.taskMutex.Unlock() - }() - - batchIdentifierHash := agg.batchesIdentifierHashByIdx[idx] - + oldTaskIdx := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] + for i := lastIdxDeleted; i <= oldTaskIdx; i++ { + batchIdentifierHash, exists := agg.batchesIdentifierHashByIdx[i] + if exists { + agg.logger.Info("Cleaning up finalized task", "taskIndex", i) delete(agg.batchesIdxByIdentifierHash, batchIdentifierHash) - delete(agg.batchCreatedBlockByIdx, idx) - delete(agg.batchesIdentifierHashByIdx, idx) + delete(agg.batchCreatedBlockByIdx, i) + delete(agg.batchesIdentifierHashByIdx, i) delete(agg.batchDataByIdentifierHash, batchIdentifierHash) - - delete(agg.batchIsFinalizedByIdx, idx) - }() + } } - + lastIdxDeleted = oldTaskIdx } } - -// called in the chan so no need for a mutex -func (agg *Aggregator) finalizeBatchIdx(idx uint32) { - agg.batchIsFinalizedByIdx[idx] = struct{}{} //now the key is present, no need to waste memory on a value -} diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index d23403b03..6e2cb0025 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -2,6 +2,7 @@ package chainio import ( "context" + "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" ethcommon "github.com/ethereum/go-ethereum/common" @@ -102,3 +103,49 @@ func (r *AvsReader) GetNotRespondedTasksFrom(fromBlock uint64) ([]servicemanager return tasks, nil } + +// This function is a helper to get a task hash of aproximately nBlocksOld blocks ago +func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { + // r.ChainReader.ethClient.CallContext(context.Background(), &blockNumber, "eth_blockNumber") + latestBlock, err := r.AvsContractBindings.ethClient.BlockNumber(context.Background()) + if err != nil { + latestBlock, err = r.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to get latest block number: %w", err) + } + } + + var fromBlock uint64 + var toBlock uint64 + + if latestBlock < nBlocksOld { + // upToBlock = latestBlock + return nil, fmt.Errorf("latest block is less than nBlocksOld") + } + interval := uint64(10) // TODO set 1000, Arbitrary number, aproximately blocks in 3hs + toBlock = latestBlock - nBlocksOld + fromBlock = toBlock - interval + // Maybe there is a way to only get 1 task? + // this is the MVP + + var lastLog *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3 + logs, err := r.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: &toBlock, Context: context.Background()}, nil) + if err != nil { + return nil, err + } + if err := logs.Error(); err != nil { + return nil, err + } + + // Any log from the list is good enough. + lastLog = logs.Event + + if lastLog == nil { + return nil, fmt.Errorf("no batches found of at least %d blocks old", interval) + } + + batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) + batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) + return &batchIdentifierHash, nil + +} From 9d8faebc8aaf52e79d88a697ba09bd1d17414cc4 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:51:34 -0300 Subject: [PATCH 17/30] fix: wip --- core/chainio/avs_reader.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index 6e2cb0025..1016d8eb1 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -115,20 +115,19 @@ func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { } } - var fromBlock uint64 - var toBlock uint64 - if latestBlock < nBlocksOld { - // upToBlock = latestBlock return nil, fmt.Errorf("latest block is less than nBlocksOld") } + + // Define block number limits to query the rpc + var fromBlock uint64 + var toBlock uint64 + interval := uint64(10) // TODO set 1000, Arbitrary number, aproximately blocks in 3hs toBlock = latestBlock - nBlocksOld - fromBlock = toBlock - interval - // Maybe there is a way to only get 1 task? - // this is the MVP + // fromBlock = toBlock - interval + fromBlock = 0 - var lastLog *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3 logs, err := r.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: &toBlock, Context: context.Background()}, nil) if err != nil { return nil, err @@ -136,16 +135,17 @@ func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { if err := logs.Error(); err != nil { return nil, err } + if !logs.Next() { + return nil, fmt.Errorf("no tasks found of at least %d blocks old", interval) + } // Any log from the list is good enough. - lastLog = logs.Event - - if lastLog == nil { - return nil, fmt.Errorf("no batches found of at least %d blocks old", interval) + task, err := r.AvsContractBindings.ServiceManager.ParseNewBatchV3(logs.Event.Raw) + if err != nil { + return nil, err } - batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) + batchIdentifier := append(task.BatchMerkleRoot[:], task.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) return &batchIdentifierHash, nil - } From 73914958aa72ae0e3b11f9a859a4f98ba429aafb Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:40:55 -0300 Subject: [PATCH 18/30] fix: cleaner --- aggregator/internal/pkg/aggregator.go | 11 +- batcher/Cargo.lock | 877 +++++++++++++------------- core/chainio/avs_reader.go | 3 +- 3 files changed, 451 insertions(+), 440 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 40d5336e0..b739287dd 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -66,7 +66,6 @@ type Aggregator struct { // Stores the TaskResponse for each batch by batchIdentifierHash batchDataByIdentifierHash map[[32]byte]BatchData - // This task index is to communicate with the local BLS // Service. // Note: In case of a reboot it can start from 0 again @@ -218,8 +217,8 @@ func (agg *Aggregator) Start(ctx context.Context) error { const MaxSentTxRetries = 5 -const garbageCollectorPeriod = time.Second * 30 //TODO change to time.Day * 1 -const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks +const garbageCollectorPeriod = time.Second * 150 //TODO change to time.Day * 1 +const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second @@ -408,7 +407,8 @@ func (agg *Aggregator) clearTasksFromMaps(period time.Duration, blocksOld uint64 } oldTaskIdx := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] - for i := lastIdxDeleted; i <= oldTaskIdx; i++ { + agg.logger.Info("Old task found", "taskIndex", oldTaskIdx) + for i := lastIdxDeleted+1; i <= oldTaskIdx; i++ { batchIdentifierHash, exists := agg.batchesIdentifierHashByIdx[i] if exists { agg.logger.Info("Cleaning up finalized task", "taskIndex", i) @@ -416,8 +416,11 @@ func (agg *Aggregator) clearTasksFromMaps(period time.Duration, blocksOld uint64 delete(agg.batchCreatedBlockByIdx, i) delete(agg.batchesIdentifierHashByIdx, i) delete(agg.batchDataByIdentifierHash, batchIdentifierHash) + } else { + agg.logger.Warn("Task not found in maps", "taskIndex", i) } } lastIdxDeleted = oldTaskIdx + agg.AggregatorConfig.BaseConfig.Logger.Info("Done cleaning finalized tasks from maps") } } diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index dea4239ba..d531431a3 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -25,19 +25,13 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -78,7 +72,7 @@ dependencies = [ [[package]] name = "aligned" -version = "0.8.0" +version = "0.9.0" dependencies = [ "aligned-sdk", "clap", @@ -179,9 +173,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a43b18702501396fa9bcdeecd533bc85fac75150d308fc0f6800a01e6234a003" +checksum = "26154390b1d205a4a7ac7352aa2eb4f81f391399d4e2f546fb81a2f8bb383f62" dependencies = [ "arrayvec", "bytes", @@ -198,7 +192,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -210,11 +204,11 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.2.6", + "indexmap 2.6.0", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "syn-solidity", "tiny-keccak", ] @@ -230,7 +224,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "syn-solidity", ] @@ -272,9 +266,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -287,33 +281,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -321,9 +315,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "ark-bn254" @@ -405,7 +399,7 @@ dependencies = [ "num-bigint 0.4.6", "num-traits", "paste", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "zeroize", ] @@ -561,15 +555,15 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ascii-canvas" @@ -582,13 +576,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -599,7 +593,7 @@ checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ "futures", "pharos", - "rustc_version 0.4.0", + "rustc_version 0.4.1", ] [[package]] @@ -616,7 +610,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -627,9 +621,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.6" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" +checksum = "7198e6f03240fdceba36656d8be440297b6b82270325908c7381f37d826a74f6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -690,14 +684,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.9.1", + "uuid 1.10.0", ] [[package]] name = "aws-sdk-s3" -version = "1.49.0" +version = "1.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e518950d4ac43508c8bfc2fe4e24b0752d99eab80134461d5e162dcda0214b55" +checksum = "cecd672c8d4265fd4fbecacd4a479180e616881bbe639250cf81ddb604e4c301" dependencies = [ "ahash", "aws-credential-types", @@ -730,9 +724,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.42.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27bf24cd0d389daa923e974b0e7c38daf308fc21e963c049f57980235017175e" +checksum = "0dc2faec3205d496c7e57eff685dd944203df7ce16a4116d0281c44021788a7b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -752,9 +746,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.43.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43b3220f1c46ac0e9dcc0a97d94b93305dacb36d1dd393996300c6b9b74364" +checksum = "c93c241f52bc5e0476e259c953234dab7e2a35ee207ee202e86c0095ec4951dc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -774,9 +768,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.42.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c46924fb1add65bba55636e12812cae2febf68c0f37361766f627ddcca91ce" +checksum = "b259429be94a3459fa1b00c5684faee118d74f9577cc50aebadc36e507c63b5f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -909,9 +903,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -922,9 +916,9 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -953,9 +947,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", "bytes", @@ -964,7 +958,7 @@ dependencies = [ "http 0.2.12", "http 1.1.0", "http-body 0.4.6", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -996,7 +990,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "tracing", ] @@ -1011,9 +1005,9 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "itoa", "matchit", @@ -1044,7 +1038,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -1057,18 +1051,18 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", "serde", + "windows-targets 0.52.6", ] [[package]] @@ -1134,9 +1128,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -1149,9 +1143,9 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", "which", ] @@ -1211,21 +1205,21 @@ checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" dependencies = [ "arrayref", "arrayvec", - "constant_time_eq 0.3.0", + "constant_time_eq 0.3.1", ] [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.3.0", - "rayon", + "constant_time_eq 0.3.1", + "rayon-core", ] [[package]] @@ -1255,7 +1249,7 @@ name = "bonsai-sdk" version = "0.8.0" source = "git+https://github.com/risc0/risc0?tag=v1.0.1#79de616506543634cb5d75b9db7f3aee3640d68c" dependencies = [ - "reqwest 0.12.7", + "reqwest 0.12.8", "risc0-groth16", "serde", "thiserror", @@ -1285,22 +1279,22 @@ checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1311,9 +1305,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" dependencies = [ "serde", ] @@ -1351,9 +1345,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -1383,9 +1377,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.20" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", @@ -1492,21 +1486,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cobs" @@ -1568,9 +1562,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "console" @@ -1612,9 +1606,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "convert_case" @@ -1634,15 +1628,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1653,7 +1647,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ - "rustc_version 0.4.0", + "rustc_version 0.4.1", ] [[package]] @@ -1749,7 +1743,7 @@ dependencies = [ "cpufeatures", "curve25519-dalek-derive", "fiat-crypto", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "subtle", "zeroize", ] @@ -1762,14 +1756,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -1777,27 +1771,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1856,8 +1850,8 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.0", - "syn 2.0.77", + "rustc_version 0.4.1", + "syn 2.0.79", ] [[package]] @@ -1950,9 +1944,9 @@ checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "ecdsa" @@ -2037,6 +2031,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "ena" version = "0.14.3" @@ -2081,9 +2081,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" dependencies = [ "log", "regex", @@ -2091,9 +2091,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" dependencies = [ "anstream", "anstyle", @@ -2294,7 +2294,7 @@ dependencies = [ "regex", "serde", "serde_json", - "syn 2.0.77", + "syn 2.0.79", "toml", "walkdir", ] @@ -2317,7 +2317,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.77", + "syn 2.0.79", "toml", "walkdir", ] @@ -2335,7 +2335,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2350,7 +2350,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2368,7 +2368,7 @@ dependencies = [ "ethabi", "generic-array 0.14.7", "k256", - "num_enum 0.7.2", + "num_enum 0.7.3", "once_cell", "open-fastrlp", "rand", @@ -2376,7 +2376,7 @@ dependencies = [ "serde", "serde_json", "strum", - "syn 2.0.77", + "syn 2.0.79", "tempfile", "thiserror", "tiny-keccak", @@ -2397,7 +2397,7 @@ dependencies = [ "ethabi", "generic-array 0.14.7", "k256", - "num_enum 0.7.2", + "num_enum 0.7.3", "once_cell", "open-fastrlp", "rand", @@ -2405,7 +2405,7 @@ dependencies = [ "serde", "serde_json", "strum", - "syn 2.0.77", + "syn 2.0.79", "tempfile", "thiserror", "tiny-keccak", @@ -2632,9 +2632,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fastrlp" @@ -2713,12 +2713,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -2727,6 +2727,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2769,9 +2775,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -2784,9 +2790,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -2794,15 +2800,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -2811,9 +2817,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-locks" @@ -2827,26 +2833,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -2860,9 +2866,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -2925,15 +2931,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "git2" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ "bitflags 2.6.0", "libc", @@ -2995,7 +3001,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3004,9 +3010,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -3014,7 +3020,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3080,6 +3086,17 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashers" version = "1.0.1" @@ -3190,9 +3207,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -3207,7 +3224,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -3231,9 +3248,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -3255,16 +3272,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -3282,7 +3299,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "log", "rustls 0.21.12", "rustls-native-certs", @@ -3292,20 +3309,20 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", - "rustls 0.23.11", + "rustls 0.23.14", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", "tower-service", - "webpki-roots 0.26.5", + "webpki-roots 0.26.6", ] [[package]] @@ -3316,7 +3333,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -3326,29 +3343,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.4.0", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3440,12 +3456,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -3482,15 +3498,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -3536,18 +3552,18 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -3605,9 +3621,9 @@ dependencies = [ [[package]] name = "keccak-asm" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47a3633291834c4fbebf8673acbc1b04ec9d151418ff9b8e26dcd79129928758" +checksum = "505d1856a39b200489082f90d897c3f07c455563880bc5952e38eabf731c83b6" dependencies = [ "digest 0.10.7", "sha3-asm", @@ -3680,15 +3696,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libgit2-sys" -version = "0.16.2+1.7.2" +version = "0.17.0+1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" dependencies = [ "cc", "libc", @@ -3698,9 +3714,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -3724,9 +3740,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -3758,11 +3774,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -3824,15 +3840,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -3844,13 +3851,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4041,11 +4049,11 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ - "num_enum_derive 0.7.2", + "num_enum_derive 0.7.3", ] [[package]] @@ -4062,14 +4070,14 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4089,9 +4097,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.1" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -4135,9 +4143,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -4156,7 +4164,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4167,9 +4175,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", @@ -4506,7 +4514,7 @@ version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "syn 1.0.109", @@ -4627,9 +4635,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.12" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -4643,7 +4651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.6.0", ] [[package]] @@ -4653,7 +4661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ "futures", - "rustc_version 0.4.0", + "rustc_version 0.4.1", ] [[package]] @@ -4686,7 +4694,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4709,22 +4717,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4767,18 +4775,19 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postcard" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" +checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" dependencies = [ "cobs", - "embedded-io", + "embedded-io 0.4.0", + "embedded-io 0.6.1", "serde", ] @@ -4790,9 +4799,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "precomputed-hash" @@ -4802,12 +4814,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4832,7 +4844,7 @@ checksum = "714c75db297bc88a63783ffc6ab9f830698a6705aa0201416931759ef4c8183d" dependencies = [ "autocfg", "equivalent", - "indexmap 2.2.6", + "indexmap 2.6.0", ] [[package]] @@ -4847,11 +4859,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit 0.22.22", ] [[package]] @@ -4880,9 +4892,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -4967,7 +4979,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4978,9 +4990,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "psm" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" dependencies = [ "cc", ] @@ -4993,16 +5005,17 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls 0.23.11", + "rustc-hash 2.0.0", + "rustls 0.23.14", + "socket2", "thiserror", "tokio", "tracing", @@ -5010,15 +5023,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", "ring 0.17.8", - "rustc-hash", - "rustls 0.23.11", + "rustc-hash 2.0.0", + "rustls 0.23.14", "slab", "thiserror", "tinyvec", @@ -5027,22 +5040,22 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -5123,18 +5136,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -5143,9 +5156,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -5205,7 +5218,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -5234,9 +5247,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", @@ -5244,12 +5257,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.4.0", - "hyper-rustls 0.27.2", + "hyper 1.4.1", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -5261,8 +5274,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.11", - "rustls-pemfile 2.1.3", + "rustls 0.23.14", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -5279,20 +5292,20 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.26.5", + "webpki-roots 0.26.6", "windows-registry", ] [[package]] name = "reqwest-middleware" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39346a33ddfe6be00cbc17a34ce996818b97b230b87229f10114693becca1268" +checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "thiserror", "tower-service", @@ -5592,6 +5605,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc-hex" version = "2.1.0" @@ -5609,18 +5628,18 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver 1.0.23", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -5643,14 +5662,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.5", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -5678,19 +5697,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -5704,9 +5722,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -5773,7 +5791,7 @@ version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d35494501194174bda522a32605929eefc9ecf7e0a326c26db1fdd85881eb62" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "syn 1.0.109", @@ -5781,20 +5799,20 @@ dependencies = [ [[package]] name = "scc" -version = "2.1.17" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c947adb109a8afce5fc9c7bf951f87f146e9147b3a6a58413105628fb1d1e66" +checksum = "553f8299af7450cda9a52d3a370199904e7a46b5ffd1bef187c4a6af3bb6db69" dependencies = [ "sdd", ] [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5833,9 +5851,9 @@ dependencies = [ [[package]] name = "sdd" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc" +checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95" [[package]] name = "sec1" @@ -5867,9 +5885,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -5880,9 +5898,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -5929,31 +5947,32 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5976,14 +5995,14 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -6002,15 +6021,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -6020,14 +6039,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6036,7 +6055,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -6065,7 +6084,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6102,9 +6121,9 @@ dependencies = [ [[package]] name = "sha3-asm" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9b57fd861253bff08bb1919e995f90ba8f4889de2726091c8876f3a4e823b40" +checksum = "c28efc5e327c837aa837c59eae585fc250715ef939ac32881bcc11677cd02d46" dependencies = [ "cc", "cfg-if", @@ -6525,7 +6544,7 @@ dependencies = [ "p3-fri", "p3-matrix", "prost", - "reqwest 0.12.7", + "reqwest 0.12.8", "reqwest-middleware", "serde", "serde_json", @@ -6577,15 +6596,15 @@ dependencies = [ [[package]] name = "stacker" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", "cfg-if", "libc", "psm", - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -6632,7 +6651,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6683,9 +6702,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -6701,7 +6720,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6784,9 +6803,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -6808,22 +6827,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6895,32 +6914,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6949,7 +6967,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.11", + "rustls 0.23.14", "rustls-pki-types", "tokio", ] @@ -6990,20 +7008,20 @@ dependencies = [ "futures-util", "log", "native-tls", - "rustls 0.23.11", + "rustls 0.23.14", "rustls-pki-types", "tokio", "tokio-native-tls", "tokio-rustls 0.26.0", "tungstenite 0.23.0", - "webpki-roots 0.26.5", + "webpki-roots 0.26.6", ] [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -7014,21 +7032,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.15", + "toml_edit 0.22.22", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -7039,33 +7057,22 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.18", + "winnow 0.6.20", ] [[package]] @@ -7086,15 +7093,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -7116,7 +7123,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7249,7 +7256,7 @@ dependencies = [ "log", "native-tls", "rand", - "rustls 0.23.11", + "rustls 0.23.14", "rustls-pki-types", "sha1", "thiserror", @@ -7268,9 +7275,9 @@ dependencies = [ "futures", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "prost", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "thiserror", @@ -7320,15 +7327,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" @@ -7347,9 +7354,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unsafe-libyaml" @@ -7410,9 +7417,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" [[package]] name = "valuable" @@ -7428,9 +7435,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.3.1" +version = "8.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" dependencies = [ "anyhow", "cfg-if", @@ -7441,9 +7448,9 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vsimd" @@ -7490,7 +7497,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "log", "mime", "mime_guess", @@ -7516,34 +7523,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -7553,9 +7561,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7563,22 +7571,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wasm-streams" @@ -7595,9 +7603,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -7621,9 +7629,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -7658,11 +7666,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7879,9 +7887,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -7907,7 +7915,7 @@ dependencies = [ "js-sys", "log", "pharos", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "send_wrapper 0.6.0", "thiserror", "wasm-bindgen", @@ -7942,6 +7950,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -7953,7 +7962,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7973,7 +7982,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8044,9 +8053,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index 1016d8eb1..ece0a7060 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -125,8 +125,7 @@ func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { interval := uint64(10) // TODO set 1000, Arbitrary number, aproximately blocks in 3hs toBlock = latestBlock - nBlocksOld - // fromBlock = toBlock - interval - fromBlock = 0 + fromBlock = toBlock - interval logs, err := r.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: &toBlock, Context: context.Background()}, nil) if err != nil { From ef8e5d67b3f52bebd53dd1a19d84f0044a02e865 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:29:50 -0300 Subject: [PATCH 19/30] refactor: GC has supervisor --- aggregator/cmd/main.go | 11 +++++++++++ aggregator/internal/pkg/aggregator.go | 14 +++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index 9a4cfed7d..177d6c3ae 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "time" "github.com/urfave/cli/v2" "github.com/yetanotherco/aligned_layer/aggregator/internal/pkg" @@ -38,6 +39,9 @@ func main() { } } +const garbageCollectorPeriod = time.Second * 150 //TODO change to time.Day * 1 +const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks + func aggregatorMain(ctx *cli.Context) error { configFilePath := ctx.String(config.ConfigFileFlag.Name) @@ -49,6 +53,13 @@ func aggregatorMain(ctx *cli.Context) error { return err } + // Supervisor revives garbage collector + go func() { + for { + aggregator.ClearTasksFromMaps(garbageCollectorPeriod, garbageCollectorTasksAge) + } + }() + // Listen for new task created in the ServiceManager contract in a separate goroutine, both V1 and V2 subscriptions: go func() { listenErr := aggregator.SubscribeToNewTasks() diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index b739287dd..c0b9a6a79 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -178,7 +178,6 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error telemetry: aggregatorTelemetry, } - go aggregator.clearTasksFromMaps(garbageCollectorPeriod, garbageCollectorTasksAge) return &aggregator, nil } @@ -217,8 +216,6 @@ func (agg *Aggregator) Start(ctx context.Context) error { const MaxSentTxRetries = 5 -const garbageCollectorPeriod = time.Second * 150 //TODO change to time.Day * 1 -const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second @@ -392,8 +389,15 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by } // long-lived gorouting that periodically checks and removes old Tasks from stored Maps -func (agg *Aggregator) clearTasksFromMaps(period time.Duration, blocksOld uint64) { - agg.AggregatorConfig.BaseConfig.Logger.Info("- Removing finalized Task Infos from Maps every %d seconds", period) +func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64) { + defer func() { + err := recover() //stops panics + if err != nil { + agg.logger.Error(err.(string)) + } + }() + + agg.AggregatorConfig.BaseConfig.Logger.Info(fmt.Sprintf("- Removing finalized Task Infos from Maps every %v", period)) lastIdxDeleted := uint32(0) for { From d9afb7b88eee10520427b71f66cbf1c82dccfcae Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:40:03 -0300 Subject: [PATCH 20/30] feat: read GC vars from config file --- aggregator/cmd/main.go | 7 +++---- aggregator/internal/pkg/aggregator.go | 6 ++---- config-files/config-aggregator.yaml | 3 +++ core/config/aggregator.go | 7 +++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index 177d6c3ae..029bb1a0e 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "time" "github.com/urfave/cli/v2" "github.com/yetanotherco/aligned_layer/aggregator/internal/pkg" @@ -39,8 +38,6 @@ func main() { } } -const garbageCollectorPeriod = time.Second * 150 //TODO change to time.Day * 1 -const garbageCollectorTasksAge = uint64(10) //TODO change to 2592000, 1 month of blocks func aggregatorMain(ctx *cli.Context) error { @@ -56,7 +53,9 @@ func aggregatorMain(ctx *cli.Context) error { // Supervisor revives garbage collector go func() { for { - aggregator.ClearTasksFromMaps(garbageCollectorPeriod, garbageCollectorTasksAge) + log.Println("Starting Garbage collector") + aggregator.ClearTasksFromMaps(aggregatorConfig.Aggregator.GarbageCollectorPeriod, aggregatorConfig.Aggregator.GarbageCollectorTasksAge) + log.Println("Garbage collector panicked, Supervisor restarting") } }() diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index c0b9a6a79..abbbf6ee9 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -178,7 +178,6 @@ func NewAggregator(aggregatorConfig config.AggregatorConfig) (*Aggregator, error telemetry: aggregatorTelemetry, } - return &aggregator, nil } @@ -216,7 +215,6 @@ func (agg *Aggregator) Start(ctx context.Context) error { const MaxSentTxRetries = 5 - const BLS_AGG_SERVICE_TIMEOUT = 100 * time.Second func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsAggregationServiceResponse) { @@ -406,13 +404,13 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(blocksOld) if err != nil { - agg.logger.Error("Error getting old task hash, skipping this garbage collect", "err", err) + agg.logger.Warn("Error getting old task hash, skipping this garbage collect", "err", err) continue // Retry in the next iteration } oldTaskIdx := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] agg.logger.Info("Old task found", "taskIndex", oldTaskIdx) - for i := lastIdxDeleted+1; i <= oldTaskIdx; i++ { + for i := lastIdxDeleted + 1; i <= oldTaskIdx; i++ { batchIdentifierHash, exists := agg.batchesIdentifierHashByIdx[i] if exists { agg.logger.Info("Cleaning up finalized task", "taskIndex", i) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index cb9f279a6..6af58bb02 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -35,6 +35,9 @@ aggregator: enable_metrics: true metrics_ip_port_address: localhost:9091 telemetry_ip_port_address: localhost:4001 + garbage_collector_period: 11s #TODO make it 1 day + garbage_collector_tasks_age: 10 #TODO make it 1 month + ## Operator Configurations # operator: # aggregator_rpc_server_ip_port_address: localhost:8090 diff --git a/core/config/aggregator.go b/core/config/aggregator.go index 740e75ae0..bf8822d9d 100644 --- a/core/config/aggregator.go +++ b/core/config/aggregator.go @@ -4,6 +4,7 @@ import ( "errors" "log" "os" + "time" sdkutils "github.com/Layr-Labs/eigensdk-go/utils" "github.com/ethereum/go-ethereum/common" @@ -20,6 +21,8 @@ type AggregatorConfig struct { EnableMetrics bool MetricsIpPortAddress string TelemetryIpPortAddress string + GarbageCollectorPeriod time.Duration + GarbageCollectorTasksAge uint64 } } @@ -31,6 +34,8 @@ type AggregatorConfigFromYaml struct { EnableMetrics bool `yaml:"enable_metrics"` MetricsIpPortAddress string `yaml:"metrics_ip_port_address"` TelemetryIpPortAddress string `yaml:"telemetry_ip_port_address"` + GarbageCollectorPeriod time.Duration `yaml:"garbage_collector_period"` + GarbageCollectorTasksAge uint64 `yaml:"garbage_collector_tasks_age"` } `yaml:"aggregator"` } @@ -72,6 +77,8 @@ func NewAggregatorConfig(configFilePath string) *AggregatorConfig { EnableMetrics bool MetricsIpPortAddress string TelemetryIpPortAddress string + GarbageCollectorPeriod time.Duration + GarbageCollectorTasksAge uint64 }(aggregatorConfigFromYaml.Aggregator), } } From d41399ed288dcdcac96750b1151fd2063abdf03a Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:41:19 -0300 Subject: [PATCH 21/30] chore: add comments --- config-files/config-aggregator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 6af58bb02..e94baf407 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -35,8 +35,8 @@ aggregator: enable_metrics: true metrics_ip_port_address: localhost:9091 telemetry_ip_port_address: localhost:4001 - garbage_collector_period: 11s #TODO make it 1 day - garbage_collector_tasks_age: 10 #TODO make it 1 month + garbage_collector_period: 11s #The period of the GC process #TODO make it 1 day + garbage_collector_tasks_age: 10 #The age of tasks that will be removed by the GC #TODO make it 1 month ## Operator Configurations # operator: From 5403e7372fc2ab562d6535687797b324676b90de Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:44:15 -0300 Subject: [PATCH 22/30] chore: simple PR comments --- aggregator/internal/pkg/aggregator.go | 6 ++++-- core/chainio/avs_reader.go | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index abbbf6ee9..b8daf4b60 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -387,6 +387,8 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by } // long-lived gorouting that periodically checks and removes old Tasks from stored Maps +// It runs every period and removes all tasks older than blocksOld +// This was added because each task occupies memory in the maps, and we need to free it to avoid a memory leak func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64) { defer func() { err := recover() //stops panics @@ -399,8 +401,6 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 lastIdxDeleted := uint32(0) for { - time.Sleep(period) - agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(blocksOld) if err != nil { @@ -424,5 +424,7 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 } lastIdxDeleted = oldTaskIdx agg.AggregatorConfig.BaseConfig.Logger.Info("Done cleaning finalized tasks from maps") + + time.Sleep(period) } } diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index ece0a7060..e226760bb 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -106,7 +106,6 @@ func (r *AvsReader) GetNotRespondedTasksFrom(fromBlock uint64) ([]servicemanager // This function is a helper to get a task hash of aproximately nBlocksOld blocks ago func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { - // r.ChainReader.ethClient.CallContext(context.Background(), &blockNumber, "eth_blockNumber") latestBlock, err := r.AvsContractBindings.ethClient.BlockNumber(context.Background()) if err != nil { latestBlock, err = r.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) From c02265b4aa5fac3d1021766ebb5459958ef3ae57 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:30:07 -0300 Subject: [PATCH 23/30] fix: nits --- aggregator/internal/pkg/aggregator.go | 19 ++++++++++++------- config-files/config-aggregator.yaml | 4 ++-- core/chainio/avs_reader.go | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index b8daf4b60..be2c1bb80 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -401,16 +401,23 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 lastIdxDeleted := uint32(0) for { + time.Sleep(period) + agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(blocksOld) if err != nil { - agg.logger.Warn("Error getting old task hash, skipping this garbage collect", "err", err) + agg.logger.Error("Error getting old task hash, skipping this garbage collect", "err", err) + continue // Retry in the next iteration + } + if oldTaskIdHash == nil { + agg.logger.Warn("No old tasks found") continue // Retry in the next iteration } - oldTaskIdx := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] - agg.logger.Info("Old task found", "taskIndex", oldTaskIdx) - for i := lastIdxDeleted + 1; i <= oldTaskIdx; i++ { + taskIdxToDelete := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] + agg.logger.Info("Old task found", "taskIndex", taskIdxToDelete) + // delete from lastIdxDeleted to taskIdxToDelete + for i := lastIdxDeleted+1; i <= taskIdxToDelete; i++ { batchIdentifierHash, exists := agg.batchesIdentifierHashByIdx[i] if exists { agg.logger.Info("Cleaning up finalized task", "taskIndex", i) @@ -422,9 +429,7 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 agg.logger.Warn("Task not found in maps", "taskIndex", i) } } - lastIdxDeleted = oldTaskIdx + lastIdxDeleted = taskIdxToDelete agg.AggregatorConfig.BaseConfig.Logger.Info("Done cleaning finalized tasks from maps") - - time.Sleep(period) } } diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index e94baf407..0b9838d9a 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -35,8 +35,8 @@ aggregator: enable_metrics: true metrics_ip_port_address: localhost:9091 telemetry_ip_port_address: localhost:4001 - garbage_collector_period: 11s #The period of the GC process #TODO make it 1 day - garbage_collector_tasks_age: 10 #The age of tasks that will be removed by the GC #TODO make it 1 month + garbage_collector_period: 60s #The period of the GC process #TODO make it 1 day + garbage_collector_tasks_age: 3 #The age of tasks that will be removed by the GC #TODO make it 1 month ## Operator Configurations # operator: diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index e226760bb..2cd050e55 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -134,7 +134,7 @@ func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { return nil, err } if !logs.Next() { - return nil, fmt.Errorf("no tasks found of at least %d blocks old", interval) + return nil, nil //not an error, but no tasks found } // Any log from the list is good enough. From 51c454d5bb134778e0d96bdb95b398d822e9bb64 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 18 Oct 2024 15:25:21 -0300 Subject: [PATCH 24/30] Unneeded whitespace --- aggregator/cmd/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index 029bb1a0e..b60c755c2 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -38,7 +38,6 @@ func main() { } } - func aggregatorMain(ctx *cli.Context) error { configFilePath := ctx.String(config.ConfigFileFlag.Name) From 79d38d81b221b5714ed5b5abd2e21f9c6a5dbe5f Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 18 Oct 2024 15:26:45 -0300 Subject: [PATCH 25/30] Typo --- aggregator/internal/pkg/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index be2c1bb80..497c3167e 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -386,7 +386,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } -// long-lived gorouting that periodically checks and removes old Tasks from stored Maps +// Long-lived goroutine that periodically checks and removes old Tasks from stored Maps // It runs every period and removes all tasks older than blocksOld // This was added because each task occupies memory in the maps, and we need to free it to avoid a memory leak func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64) { From 74add84cbe0b276537b199431b1ce87051064864 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Fri, 18 Oct 2024 15:29:02 -0300 Subject: [PATCH 26/30] Use `.Error()` rather than `.(string)` --- aggregator/internal/pkg/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 497c3167e..9e5f4bff2 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -393,7 +393,7 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 defer func() { err := recover() //stops panics if err != nil { - agg.logger.Error(err.(string)) + agg.logger.Error(err.Error()) } }() From 6919a23969e15590f91b49781caf368ead12c667 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:25:24 -0300 Subject: [PATCH 27/30] fix: print error --- aggregator/internal/pkg/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 9e5f4bff2..8d69c5a6f 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -393,7 +393,7 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 defer func() { err := recover() //stops panics if err != nil { - agg.logger.Error(err.Error()) + agg.logger.Error("Recovered from panic", "err", err) } }() From 45e49a045b3750d200699d36856ce62d903e4d56 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:57:38 -0300 Subject: [PATCH 28/30] chore: set numbers for prod --- config-files/config-aggregator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 0b9838d9a..49ec23c5e 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -35,8 +35,8 @@ aggregator: enable_metrics: true metrics_ip_port_address: localhost:9091 telemetry_ip_port_address: localhost:4001 - garbage_collector_period: 60s #The period of the GC process #TODO make it 1 day - garbage_collector_tasks_age: 3 #The age of tasks that will be removed by the GC #TODO make it 1 month + garbage_collector_period: 5m #The period of the GC process. Suggested value for Prod: '168h' (7 days) + garbage_collector_tasks_age: 50 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) ## Operator Configurations # operator: From 7f9ddb00c929117033baafd5ad5dca4334470c3a Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:02:24 -0300 Subject: [PATCH 29/30] chore: set better values for dev --- config-files/config-aggregator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 49ec23c5e..4debee412 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -35,8 +35,8 @@ aggregator: enable_metrics: true metrics_ip_port_address: localhost:9091 telemetry_ip_port_address: localhost:4001 - garbage_collector_period: 5m #The period of the GC process. Suggested value for Prod: '168h' (7 days) - garbage_collector_tasks_age: 50 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) + garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days) + garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) ## Operator Configurations # operator: From 3c1898c268a03e0a8a932109963b882a6da3bd52 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:44:16 -0300 Subject: [PATCH 30/30] refcator: GarbageCollectorTasksInterval to config.yml and remove params of call --- aggregator/cmd/main.go | 2 +- aggregator/internal/pkg/aggregator.go | 12 ++++++------ config-files/config-aggregator.yaml | 1 + core/chainio/avs_reader.go | 6 ++---- core/config/aggregator.go | 3 +++ 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index b60c755c2..4e02ff206 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -53,7 +53,7 @@ func aggregatorMain(ctx *cli.Context) error { go func() { for { log.Println("Starting Garbage collector") - aggregator.ClearTasksFromMaps(aggregatorConfig.Aggregator.GarbageCollectorPeriod, aggregatorConfig.Aggregator.GarbageCollectorTasksAge) + aggregator.ClearTasksFromMaps() log.Println("Garbage collector panicked, Supervisor restarting") } }() diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 8d69c5a6f..6dc469c09 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -387,9 +387,9 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by } // Long-lived goroutine that periodically checks and removes old Tasks from stored Maps -// It runs every period and removes all tasks older than blocksOld +// It runs every GarbageCollectorPeriod and removes all tasks older than GarbageCollectorTasksAge // This was added because each task occupies memory in the maps, and we need to free it to avoid a memory leak -func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64) { +func (agg *Aggregator) ClearTasksFromMaps() { defer func() { err := recover() //stops panics if err != nil { @@ -397,14 +397,14 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 } }() - agg.AggregatorConfig.BaseConfig.Logger.Info(fmt.Sprintf("- Removing finalized Task Infos from Maps every %v", period)) + agg.AggregatorConfig.BaseConfig.Logger.Info(fmt.Sprintf("- Removing finalized Task Infos from Maps every %v", agg.AggregatorConfig.Aggregator.GarbageCollectorPeriod)) lastIdxDeleted := uint32(0) for { - time.Sleep(period) + time.Sleep(agg.AggregatorConfig.Aggregator.GarbageCollectorPeriod) agg.AggregatorConfig.BaseConfig.Logger.Info("Cleaning finalized tasks from maps") - oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(blocksOld) + oldTaskIdHash, err := agg.avsReader.GetOldTaskHash(agg.AggregatorConfig.Aggregator.GarbageCollectorTasksAge, agg.AggregatorConfig.Aggregator.GarbageCollectorTasksInterval) if err != nil { agg.logger.Error("Error getting old task hash, skipping this garbage collect", "err", err) continue // Retry in the next iteration @@ -417,7 +417,7 @@ func (agg *Aggregator) ClearTasksFromMaps(period time.Duration, blocksOld uint64 taskIdxToDelete := agg.batchesIdxByIdentifierHash[*oldTaskIdHash] agg.logger.Info("Old task found", "taskIndex", taskIdxToDelete) // delete from lastIdxDeleted to taskIdxToDelete - for i := lastIdxDeleted+1; i <= taskIdxToDelete; i++ { + for i := lastIdxDeleted + 1; i <= taskIdxToDelete; i++ { batchIdentifierHash, exists := agg.batchesIdentifierHashByIdx[i] if exists { agg.logger.Info("Cleaning up finalized task", "taskIndex", i) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 4debee412..83ff4e49d 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -37,6 +37,7 @@ aggregator: telemetry_ip_port_address: localhost:4001 garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days) garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) + garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours) ## Operator Configurations # operator: diff --git a/core/chainio/avs_reader.go b/core/chainio/avs_reader.go index 2cd050e55..99d9c37d0 100644 --- a/core/chainio/avs_reader.go +++ b/core/chainio/avs_reader.go @@ -105,7 +105,7 @@ func (r *AvsReader) GetNotRespondedTasksFrom(fromBlock uint64) ([]servicemanager } // This function is a helper to get a task hash of aproximately nBlocksOld blocks ago -func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { +func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64, interval uint64) (*[32]byte, error) { latestBlock, err := r.AvsContractBindings.ethClient.BlockNumber(context.Background()) if err != nil { latestBlock, err = r.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) @@ -120,10 +120,8 @@ func (r *AvsReader) GetOldTaskHash(nBlocksOld uint64) (*[32]byte, error) { // Define block number limits to query the rpc var fromBlock uint64 - var toBlock uint64 - interval := uint64(10) // TODO set 1000, Arbitrary number, aproximately blocks in 3hs - toBlock = latestBlock - nBlocksOld + toBlock := latestBlock - nBlocksOld fromBlock = toBlock - interval logs, err := r.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: &toBlock, Context: context.Background()}, nil) diff --git a/core/config/aggregator.go b/core/config/aggregator.go index bf8822d9d..e38e76b78 100644 --- a/core/config/aggregator.go +++ b/core/config/aggregator.go @@ -23,6 +23,7 @@ type AggregatorConfig struct { TelemetryIpPortAddress string GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 + GarbageCollectorTasksInterval uint64 } } @@ -36,6 +37,7 @@ type AggregatorConfigFromYaml struct { TelemetryIpPortAddress string `yaml:"telemetry_ip_port_address"` GarbageCollectorPeriod time.Duration `yaml:"garbage_collector_period"` GarbageCollectorTasksAge uint64 `yaml:"garbage_collector_tasks_age"` + GarbageCollectorTasksInterval uint64 `yaml:"garbage_collector_tasks_interval"` } `yaml:"aggregator"` } @@ -79,6 +81,7 @@ func NewAggregatorConfig(configFilePath string) *AggregatorConfig { TelemetryIpPortAddress string GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 + GarbageCollectorTasksInterval uint64 }(aggregatorConfigFromYaml.Aggregator), } }