diff --git a/changelog.md b/changelog.md index 1e471a41cf..88bb15b57a 100644 --- a/changelog.md +++ b/changelog.md @@ -29,6 +29,7 @@ * [3253](https://github.com/zeta-chain/node/pull/3253) - fix solana inbound version 0 queries and move tss keysign prior to relayer key checking * [3278](https://github.com/zeta-chain/node/pull/3278) - enforce checksum format for asset address in ZRC20 * [3289](https://github.com/zeta-chain/node/pull/3289) - remove all dynamic peer discovery (zetaclient) +* [3314](https://github.com/zeta-chain/node/pull/3314) - update `last_scanned_block_number` metrics more frequently for Solana chain ## v23.0.0 diff --git a/zetaclient/chains/base/observer.go b/zetaclient/chains/base/observer.go index 8cb3ab7cdb..5afb1cae11 100644 --- a/zetaclient/chains/base/observer.go +++ b/zetaclient/chains/base/observer.go @@ -57,9 +57,6 @@ type Observer struct { // lastTxScanned is the last transaction hash scanned by the observer lastTxScanned string - // rpcAlertLatency is the threshold of RPC latency to trigger an alert - rpcAlertLatency time.Duration - blockCache *lru.Cache // db is the database to persist data @@ -87,7 +84,6 @@ func NewObserver( zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, blockCacheSize int, - rpcAlertLatency int64, ts *metrics.TelemetryServer, database *db.DB, logger Logger, @@ -105,7 +101,6 @@ func NewObserver( lastBlock: 0, lastBlockScanned: 0, lastTxScanned: "", - rpcAlertLatency: time.Duration(rpcAlertLatency) * time.Second, ts: ts, db: database, blockCache: blockCache, @@ -429,29 +424,12 @@ func (ob *Observer) PostVoteInbound( return ballot, nil } -// AlertOnRPCLatency prints an alert if the RPC latency exceeds the threshold. -// Returns true if the RPC latency is too high. -func (ob *Observer) AlertOnRPCLatency(latestBlockTime time.Time, defaultAlertLatency time.Duration) bool { +// ReportBlockLatency records the latency between the current time +// an the latest block time for a chain as a metric +func (ob *Observer) ReportBlockLatency(latestBlockTime time.Time) { elapsedTime := time.Since(latestBlockTime) - alertLatency := ob.rpcAlertLatency - if alertLatency == 0 { - alertLatency = defaultAlertLatency - } - - lf := map[string]any{ - "rpc_latency_alert_ms": alertLatency.Milliseconds(), - "rpc_latency_real_ms": elapsedTime.Milliseconds(), - } - - if elapsedTime > alertLatency { - ob.logger.Chain.Error().Fields(lf).Msg("RPC latency is too high, please check the node or explorer") - return true - } - - ob.logger.Chain.Info().Fields(lf).Msg("RPC latency is OK") - - return false + metrics.LatestBlockLatency.WithLabelValues(ob.chain.Name).Set(elapsedTime.Seconds()) } // EnvVarLatestBlockByChain returns the environment variable for the last block by chain. diff --git a/zetaclient/chains/base/observer_test.go b/zetaclient/chains/base/observer_test.go index 5c08563b7b..ab5e6fb2d6 100644 --- a/zetaclient/chains/base/observer_test.go +++ b/zetaclient/chains/base/observer_test.go @@ -6,7 +6,6 @@ import ( "os" "strings" "testing" - "time" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -25,9 +24,6 @@ import ( ) const ( - // defaultAlertLatency is the default alert latency (in seconds) for unit tests - defaultAlertLatency = 60 - // defaultConfirmationCount is the default confirmation count for unit tests defaultConfirmationCount = 2 ) @@ -40,7 +36,7 @@ type testSuite struct { } // newTestSuite creates a new observer for testing -func newTestSuite(t *testing.T, chain chains.Chain, alertLatency int64) *testSuite { +func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { // constructor parameters chainParams := *sample.ChainParams(chain.ChainId) chainParams.ConfirmationCount = defaultConfirmationCount @@ -57,7 +53,6 @@ func newTestSuite(t *testing.T, chain chains.Chain, alertLatency int64) *testSui zetacoreClient, tss, base.DefaultBlockCacheSize, - alertLatency, nil, database, logger, @@ -127,7 +122,6 @@ func TestNewObserver(t *testing.T) { tt.zetacoreClient, tt.tss, tt.blockCacheSize, - 60, nil, database, base.DefaultLogger(), @@ -147,7 +141,7 @@ func TestNewObserver(t *testing.T) { func TestStop(t *testing.T) { t.Run("should be able to stop observer", func(t *testing.T) { // create observer and initialize db - ob := newTestSuite(t, chains.Ethereum, defaultAlertLatency) + ob := newTestSuite(t, chains.Ethereum) // stop observer ob.Stop() @@ -158,7 +152,7 @@ func TestObserverGetterAndSetter(t *testing.T) { chain := chains.Ethereum t.Run("should be able to update last block", func(t *testing.T) { - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // update last block newLastBlock := uint64(100) @@ -167,7 +161,7 @@ func TestObserverGetterAndSetter(t *testing.T) { }) t.Run("should be able to update last block scanned", func(t *testing.T) { - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // update last block scanned newLastBlockScanned := uint64(100) @@ -176,7 +170,7 @@ func TestObserverGetterAndSetter(t *testing.T) { }) t.Run("should be able to update last tx scanned", func(t *testing.T) { - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // update last tx scanned newLastTxScanned := sample.EthAddress().String() @@ -185,7 +179,7 @@ func TestObserverGetterAndSetter(t *testing.T) { }) t.Run("should be able to get logger", func(t *testing.T) { - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) logger := ob.Logger() // should be able to print log @@ -233,7 +227,7 @@ func TestTSSAddressString(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // create observer - ob := newTestSuite(t, tt.chain, defaultAlertLatency) + ob := newTestSuite(t, tt.chain) // get TSS address addr := ob.TSSAddressString() @@ -286,7 +280,7 @@ func TestIsBlockConfirmed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // create observer - ob := newTestSuite(t, tt.chain, defaultAlertLatency) + ob := newTestSuite(t, tt.chain) ob.Observer.WithLastBlock(tt.lastBlock) // check if block is confirmed @@ -318,7 +312,7 @@ func TestOutboundID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // create observer - ob := newTestSuite(t, tt.chain, defaultAlertLatency) + ob := newTestSuite(t, tt.chain) // get outbound id outboundID := ob.OutboundID(tt.nonce) @@ -336,7 +330,7 @@ func TestLoadLastBlockScanned(t *testing.T) { t.Run("should be able to load last block scanned", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // create db and write 100 as last block scanned err := ob.WriteLastBlockScannedToDB(100) @@ -350,7 +344,7 @@ func TestLoadLastBlockScanned(t *testing.T) { t.Run("latest block scanned should be 0 if not found in db", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // read last block scanned err := ob.LoadLastBlockScanned(log.Logger) @@ -360,7 +354,7 @@ func TestLoadLastBlockScanned(t *testing.T) { t.Run("should overwrite last block scanned if env var is set", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // create db and write 100 as last block scanned ob.WriteLastBlockScannedToDB(100) @@ -376,7 +370,7 @@ func TestLoadLastBlockScanned(t *testing.T) { t.Run("last block scanned should remain 0 if env var is set to latest", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // create db and write 100 as last block scanned ob.WriteLastBlockScannedToDB(100) @@ -392,7 +386,7 @@ func TestLoadLastBlockScanned(t *testing.T) { t.Run("should return error on invalid env var", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // set invalid env var os.Setenv(envvar, "invalid") @@ -406,7 +400,7 @@ func TestLoadLastBlockScanned(t *testing.T) { func TestSaveLastBlockScanned(t *testing.T) { t.Run("should be able to save last block scanned", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chains.Ethereum, defaultAlertLatency) + ob := newTestSuite(t, chains.Ethereum) // save 100 as last block scanned err := ob.SaveLastBlockScanned(100) @@ -426,7 +420,7 @@ func TestReadWriteDBLastBlockScanned(t *testing.T) { chain := chains.Ethereum t.Run("should be able to write and read last block scanned to db", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // write last block scanned err := ob.WriteLastBlockScannedToDB(100) @@ -439,7 +433,7 @@ func TestReadWriteDBLastBlockScanned(t *testing.T) { t.Run("should return error when last block scanned not found in db", func(t *testing.T) { // create empty db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) lastScannedBlock, err := ob.ReadLastBlockScannedFromDB() require.Error(t, err) @@ -453,7 +447,7 @@ func TestLoadLastTxScanned(t *testing.T) { t.Run("should be able to load last tx scanned", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // create db and write sample hash as last tx scanned ob.WriteLastTxScannedToDB(lastTx) @@ -465,7 +459,7 @@ func TestLoadLastTxScanned(t *testing.T) { t.Run("latest tx scanned should be empty if not found in db", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // read last tx scanned ob.LoadLastTxScanned() @@ -474,7 +468,7 @@ func TestLoadLastTxScanned(t *testing.T) { t.Run("should overwrite last tx scanned if env var is set", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // create db and write sample hash as last tx scanned ob.WriteLastTxScannedToDB(lastTx) @@ -493,7 +487,7 @@ func TestSaveLastTxScanned(t *testing.T) { chain := chains.SolanaDevnet t.Run("should be able to save last tx scanned", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // save random tx hash lastSlot := uint64(100) @@ -516,7 +510,7 @@ func TestReadWriteDBLastTxScanned(t *testing.T) { chain := chains.SolanaDevnet t.Run("should be able to write and read last tx scanned to db", func(t *testing.T) { // create observer and open db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) // write last tx scanned lastTx := "5LuQMorgd11p8GWEw6pmyHCDtA26NUyeNFhLWPNk2oBoM9pkag1LzhwGSRos3j4TJLhKjswFhZkGtvSGdLDkmqsk" @@ -530,7 +524,7 @@ func TestReadWriteDBLastTxScanned(t *testing.T) { t.Run("should return error when last tx scanned not found in db", func(t *testing.T) { // create empty db - ob := newTestSuite(t, chain, defaultAlertLatency) + ob := newTestSuite(t, chain) lastTxScanned, err := ob.ReadLastTxScannedFromDB() require.Error(t, err) @@ -541,7 +535,7 @@ func TestReadWriteDBLastTxScanned(t *testing.T) { func TestPostVoteInbound(t *testing.T) { t.Run("should be able to post vote inbound", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.Ethereum, defaultAlertLatency) + ob := newTestSuite(t, chains.Ethereum) ob.zetacore.WithPostVoteInbound("", "sampleBallotIndex") @@ -554,7 +548,7 @@ func TestPostVoteInbound(t *testing.T) { t.Run("should not post vote if message basic validation fails", func(t *testing.T) { // create observer - ob := newTestSuite(t, chains.Ethereum, defaultAlertLatency) + ob := newTestSuite(t, chains.Ethereum) // create sample message with long Message msg := sample.InboundVote(coin.CoinType_Gas, chains.Ethereum.ChainId, chains.ZetaChainMainnet.ChainId) @@ -567,53 +561,6 @@ func TestPostVoteInbound(t *testing.T) { }) } -func TestAlertOnRPCLatency(t *testing.T) { - now := time.Now() - - tests := []struct { - name string - blockTime time.Time - alertLatency int64 - alerted bool - }{ - { - name: "should alert on high RPC latency", - blockTime: now.Add(-60 * time.Second), - alertLatency: 55, - alerted: true, - }, - { - name: "should not alert on normal RPC latency", - blockTime: now.Add(-60 * time.Second), - alertLatency: 65, - alerted: false, - }, - { - name: "should alert on higher RPC latency then default", - blockTime: now.Add(-65 * time.Second), - alertLatency: 0, // 0 means not set - alerted: true, - }, - { - name: "should not alert on normal RPC latency when compared to default", - blockTime: now.Add(-55 * time.Second), - alertLatency: 0, // 0 means not set - alerted: false, - }, - } - - // run tests - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // create observer - ob := newTestSuite(t, chains.Ethereum, tt.alertLatency) - - alerted := ob.AlertOnRPCLatency(tt.blockTime, time.Duration(defaultAlertLatency)*time.Second) - require.Equal(t, tt.alerted, alerted) - }) - } -} - func createDatabase(t *testing.T) *db.DB { sqlDatabase, err := db.NewFromSqliteInMemory(true) require.NoError(t, err) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 74bc263e3a..c3f09b17d4 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -95,7 +95,6 @@ func NewObserver( chainParams observertypes.ChainParams, zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, - rpcAlertLatency int64, database *db.DB, logger base.Logger, ts *metrics.TelemetryServer, @@ -107,7 +106,6 @@ func NewObserver( zetacoreClient, tss, btcBlocksPerDay, - rpcAlertLatency, ts, database, logger, diff --git a/zetaclient/chains/bitcoin/observer/observer_test.go b/zetaclient/chains/bitcoin/observer/observer_test.go index 18358703b5..ab5415ef66 100644 --- a/zetaclient/chains/bitcoin/observer/observer_test.go +++ b/zetaclient/chains/bitcoin/observer/observer_test.go @@ -144,7 +144,6 @@ func Test_NewObserver(t *testing.T) { tt.chainParams, tt.coreClient, tt.tss, - 60, database, tt.logger, tt.ts, @@ -326,7 +325,6 @@ func newTestSuite(t *testing.T, chain chains.Chain) *testSuite { chainParams, zetacore, nil, - 60, database, base.Logger{Std: log, Compliance: log}, nil, diff --git a/zetaclient/chains/bitcoin/observer/outbound_test.go b/zetaclient/chains/bitcoin/observer/outbound_test.go index 98b34ea435..fba95b7df3 100644 --- a/zetaclient/chains/bitcoin/observer/outbound_test.go +++ b/zetaclient/chains/bitcoin/observer/outbound_test.go @@ -43,7 +43,7 @@ func MockBTCObserverMainnet(t *testing.T, tss interfaces.TSSSigner) *Observer { baseLogger := base.Logger{Std: logger, Compliance: logger} // create Bitcoin observer - ob, err := NewObserver(chain, btcClient, params, nil, tss, 60, database, baseLogger, nil) + ob, err := NewObserver(chain, btcClient, params, nil, tss, database, baseLogger, nil) require.NoError(t, err) return ob diff --git a/zetaclient/chains/bitcoin/observer/rpc_status.go b/zetaclient/chains/bitcoin/observer/rpc_status.go index 136882b9b9..4cd935c88c 100644 --- a/zetaclient/chains/bitcoin/observer/rpc_status.go +++ b/zetaclient/chains/bitcoin/observer/rpc_status.go @@ -41,6 +41,5 @@ func (ob *Observer) checkRPCStatus() { return } - // alert if RPC latency is too high - ob.AlertOnRPCLatency(blockTime, rpc.RPCAlertLatency) + ob.ReportBlockLatency(blockTime) } diff --git a/zetaclient/chains/evm/observer/observer.go b/zetaclient/chains/evm/observer/observer.go index 79ffd40545..1cab216648 100644 --- a/zetaclient/chains/evm/observer/observer.go +++ b/zetaclient/chains/evm/observer/observer.go @@ -69,7 +69,6 @@ func NewObserver( chainParams observertypes.ChainParams, zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, - rpcAlertLatency int64, database *db.DB, logger base.Logger, ts *metrics.TelemetryServer, @@ -81,7 +80,6 @@ func NewObserver( zetacoreClient, tss, base.DefaultBlockCacheSize, - rpcAlertLatency, ts, database, logger, diff --git a/zetaclient/chains/evm/observer/observer_test.go b/zetaclient/chains/evm/observer/observer_test.go index 10fbaced9f..824ee94bb7 100644 --- a/zetaclient/chains/evm/observer/observer_test.go +++ b/zetaclient/chains/evm/observer/observer_test.go @@ -184,7 +184,6 @@ func Test_NewObserver(t *testing.T) { tt.chainParams, zetacoreClient, tt.tss, - 60, database, tt.logger, tt.ts, @@ -434,7 +433,6 @@ func newTestSuite(t *testing.T, opts ...func(*testSuiteConfig)) *testSuite { chainParams, zetacore, tss, - 60, database, logger, nil, diff --git a/zetaclient/chains/evm/observer/rpc_status.go b/zetaclient/chains/evm/observer/rpc_status.go index 68c7629523..3cb279e183 100644 --- a/zetaclient/chains/evm/observer/rpc_status.go +++ b/zetaclient/chains/evm/observer/rpc_status.go @@ -36,6 +36,5 @@ func (ob *Observer) checkRPCStatus(ctx context.Context) { return } - // alert if RPC latency is too high - ob.AlertOnRPCLatency(blockTime, rpc.RPCAlertLatency) + ob.ReportBlockLatency(blockTime) } diff --git a/zetaclient/chains/evm/signer/signer_test.go b/zetaclient/chains/evm/signer/signer_test.go index bcd1114b80..70bd04cc3b 100644 --- a/zetaclient/chains/evm/signer/signer_test.go +++ b/zetaclient/chains/evm/signer/signer_test.go @@ -97,7 +97,6 @@ func getNewEvmChainObserver(t *testing.T, tss interfaces.TSSSigner) (*observer.O params, mocks.NewZetacoreClient(t), tss, - 60, database, logger, ts, diff --git a/zetaclient/chains/solana/observer/inbound.go b/zetaclient/chains/solana/observer/inbound.go index 6eab1242ac..6cc1c8d84c 100644 --- a/zetaclient/chains/solana/observer/inbound.go +++ b/zetaclient/chains/solana/observer/inbound.go @@ -81,6 +81,12 @@ func (ob *Observer) ObserveInbound(ctx context.Context) error { ob.WithLastTxScanned(lastSig.String()) } + // query last finalized slot + lastSlot, errSlot := ob.solClient.GetSlot(ctx, rpc.CommitmentFinalized) + if errSlot != nil { + ob.Logger().Inbound.Err(errSlot).Msg("unable to get last slot") + } + // get all signatures for the gateway address since last scanned signature lastSig := solana.MustSignatureFromBase58(ob.LastTxScanned()) signatures, err := solanarpc.GetSignaturesForAddressUntil(ctx, ob.solClient, ob.gatewayID, lastSig, pageLimit) @@ -88,7 +94,13 @@ func (ob *Observer) ObserveInbound(ctx context.Context) error { ob.Logger().Inbound.Err(err).Msg("error GetSignaturesForAddressUntil") return err } - if len(signatures) > 0 { + + // update metrics if no new signatures found + if len(signatures) == 0 { + if errSlot == nil { + ob.WithLastBlockScanned(lastSlot) + } + } else { ob.Logger().Inbound.Info().Msgf("ObserveInbound: got %d signatures for chain %d", len(signatures), chainID) } diff --git a/zetaclient/chains/solana/observer/inbound_test.go b/zetaclient/chains/solana/observer/inbound_test.go index b0ed202310..6ac1110404 100644 --- a/zetaclient/chains/solana/observer/inbound_test.go +++ b/zetaclient/chains/solana/observer/inbound_test.go @@ -46,7 +46,6 @@ func Test_FilterInboundEventAndVote(t *testing.T) { *chainParams, zetacoreClient, nil, - 60, database, base.DefaultLogger(), nil, @@ -73,7 +72,7 @@ func Test_FilterInboundEvents(t *testing.T) { chainParams := sample.ChainParams(chain.ChainId) chainParams.GatewayAddress = testutils.OldSolanaGatewayAddressDevnet - ob, err := observer.NewObserver(chain, nil, *chainParams, nil, nil, 60, database, base.DefaultLogger(), nil) + ob, err := observer.NewObserver(chain, nil, *chainParams, nil, nil, database, base.DefaultLogger(), nil) require.NoError(t, err) // expected result @@ -114,7 +113,7 @@ func Test_BuildInboundVoteMsgFromEvent(t *testing.T) { database, err := db.NewFromSqliteInMemory(true) require.NoError(t, err) - ob, err := observer.NewObserver(chain, nil, *params, zetacoreClient, nil, 60, database, base.DefaultLogger(), nil) + ob, err := observer.NewObserver(chain, nil, *params, zetacoreClient, nil, database, base.DefaultLogger(), nil) require.NoError(t, err) // create test compliance config diff --git a/zetaclient/chains/solana/observer/observer.go b/zetaclient/chains/solana/observer/observer.go index 0a9b2dcc3b..56c8146825 100644 --- a/zetaclient/chains/solana/observer/observer.go +++ b/zetaclient/chains/solana/observer/observer.go @@ -44,7 +44,6 @@ func NewObserver( chainParams observertypes.ChainParams, zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, - rpcAlertLatency int64, db *db.DB, logger base.Logger, ts *metrics.TelemetryServer, @@ -56,7 +55,6 @@ func NewObserver( zetacoreClient, tss, base.DefaultBlockCacheSize, - rpcAlertLatency, ts, db, logger, diff --git a/zetaclient/chains/solana/observer/observer_test.go b/zetaclient/chains/solana/observer/observer_test.go index 5e9fdeb4b5..e5f3d195fc 100644 --- a/zetaclient/chains/solana/observer/observer_test.go +++ b/zetaclient/chains/solana/observer/observer_test.go @@ -45,7 +45,6 @@ func MockSolanaObserver( chainParams, zetacoreClient, tss, - 60, database, base.DefaultLogger(), nil, diff --git a/zetaclient/chains/solana/observer/outbound_test.go b/zetaclient/chains/solana/observer/outbound_test.go index 699253ff3d..ee51a81db5 100644 --- a/zetaclient/chains/solana/observer/outbound_test.go +++ b/zetaclient/chains/solana/observer/outbound_test.go @@ -61,7 +61,7 @@ func createTestObserver( // create observer chainParams := sample.ChainParams(chain.ChainId) chainParams.GatewayAddress = GatewayAddressTest - ob, err := observer.NewObserver(chain, solClient, *chainParams, nil, tss, 60, database, logger, nil) + ob, err := observer.NewObserver(chain, solClient, *chainParams, nil, tss, database, logger, nil) require.NoError(t, err) return ob diff --git a/zetaclient/chains/solana/observer/rpc_status.go b/zetaclient/chains/solana/observer/rpc_status.go index 1b16492076..e8f2842ec3 100644 --- a/zetaclient/chains/solana/observer/rpc_status.go +++ b/zetaclient/chains/solana/observer/rpc_status.go @@ -40,6 +40,5 @@ func (ob *Observer) checkRPCStatus(ctx context.Context) { return } - // alert if RPC latency is too high - ob.AlertOnRPCLatency(blockTime, rpc.RPCAlertLatency) + ob.ReportBlockLatency(blockTime) } diff --git a/zetaclient/chains/ton/observer/observer.go b/zetaclient/chains/ton/observer/observer.go index 565a544073..24116e7fb6 100644 --- a/zetaclient/chains/ton/observer/observer.go +++ b/zetaclient/chains/ton/observer/observer.go @@ -186,10 +186,7 @@ func (ob *Observer) checkRPCStatus(ctx context.Context) error { blockTime := time.Unix(int64(block.GenUtime), 0).UTC() - // will be overridden by chain config - const defaultAlertLatency = 30 * time.Second - - ob.AlertOnRPCLatency(blockTime, defaultAlertLatency) + ob.ReportBlockLatency(blockTime) return nil } diff --git a/zetaclient/chains/ton/observer/observer_test.go b/zetaclient/chains/ton/observer/observer_test.go index 808277c226..ea9c92e3ba 100644 --- a/zetaclient/chains/ton/observer/observer_test.go +++ b/zetaclient/chains/ton/observer/observer_test.go @@ -81,7 +81,6 @@ func newTestSuite(t *testing.T) *testSuite { zetacore, tss, 1, - 60, nil, database, logger, diff --git a/zetaclient/config/config_chain.go b/zetaclient/config/config_chain.go index 81cd3b2e1c..4328f1630e 100644 --- a/zetaclient/config/config_chain.go +++ b/zetaclient/config/config_chain.go @@ -34,26 +34,23 @@ func bitcoinConfigRegnet() BTCConfig { return BTCConfig{ // `smoketest` is the previous name for E2E test, // we keep this name for compatibility between client versions in upgrade test - RPCUsername: "smoketest", - RPCPassword: "123", - RPCHost: "bitcoin:18443", - RPCParams: "regtest", - RPCAlertLatency: 60, + RPCUsername: "smoketest", + RPCPassword: "123", + RPCHost: "bitcoin:18443", + RPCParams: "regtest", } } // solanaConfigLocalnet contains config for Solana localnet func solanaConfigLocalnet() SolanaConfig { return SolanaConfig{ - Endpoint: "http://solana:8899", - RPCAlertLatency: 60, + Endpoint: "http://solana:8899", } } func tonConfigLocalnet() TONConfig { return TONConfig{ LiteClientConfigURL: "http://ton:8000/lite-client.json", - RPCAlertLatency: 60, } } @@ -62,8 +59,7 @@ func tonConfigLocalnet() TONConfig { func evmChainsConfigs() map[int64]EVMConfig { return map[int64]EVMConfig{ chains.GoerliLocalnet.ChainId: { - Endpoint: "http://eth:8545", - RPCAlertLatency: 60, + Endpoint: "http://eth:8545", }, } } diff --git a/zetaclient/config/types.go b/zetaclient/config/types.go index 57b357e458..7e1fdddfdd 100644 --- a/zetaclient/config/types.go +++ b/zetaclient/config/types.go @@ -38,31 +38,27 @@ type ClientConfiguration struct { // EVMConfig is the config for EVM chain type EVMConfig struct { - Endpoint string `mask:"filled"` - RPCAlertLatency int64 + Endpoint string `mask:"filled"` } // BTCConfig is the config for Bitcoin chain type BTCConfig struct { // the following are rpcclient ConnConfig fields - RPCUsername string `mask:"filled"` - RPCPassword string `mask:"filled"` - RPCHost string `mask:"filled"` - RPCParams string // "regtest", "mainnet", "testnet3" , "signet", "testnet4" - RPCAlertLatency int64 + RPCUsername string `mask:"filled"` + RPCPassword string `mask:"filled"` + RPCHost string `mask:"filled"` + RPCParams string // "regtest", "mainnet", "testnet3" , "signet", "testnet4" } // SolanaConfig is the config for Solana chain type SolanaConfig struct { - Endpoint string `mask:"filled"` - RPCAlertLatency int64 + Endpoint string `mask:"filled"` } // TONConfig is the config for TON chain type TONConfig struct { // Can be either URL of local file path LiteClientConfigURL string `json:"liteClientConfigURL"` - RPCAlertLatency int64 `json:"rpcAlertLatency"` } // ComplianceConfig is the config for compliance diff --git a/zetaclient/metrics/metrics.go b/zetaclient/metrics/metrics.go index 1c2e03a21f..28fe897504 100644 --- a/zetaclient/metrics/metrics.go +++ b/zetaclient/metrics/metrics.go @@ -79,6 +79,13 @@ var ( Help: "Last scanned block number per chain", }, []string{"chain"}) + // LatestBlockLatency is a gauge that contains the block latency for each observed chain + LatestBlockLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ZetaClientNamespace, + Name: "latest_block_latency", + Help: "Latency of last block for observed chains", + }, []string{"chain"}) + // LastCoreBlockNumber is a gauge that contains the last core block number LastCoreBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: ZetaClientNamespace, diff --git a/zetaclient/orchestrator/bootstap_test.go b/zetaclient/orchestrator/bootstap_test.go index 0e36d4721d..71b3ec744b 100644 --- a/zetaclient/orchestrator/bootstap_test.go +++ b/zetaclient/orchestrator/bootstap_test.go @@ -216,7 +216,7 @@ func TestCreateChainObserverMap(t *testing.T) { _, solConfig := testrpc.NewSolanaServer(t) // Given TON config - tonConfig := config.TONConfig{LiteClientConfigURL: tonMainnet, RPCAlertLatency: 1} + tonConfig := config.TONConfig{LiteClientConfigURL: tonMainnet} // Given a zetaclient config with ETH, MATIC, and BTC chains cfg := config.New(false) diff --git a/zetaclient/orchestrator/bootstrap.go b/zetaclient/orchestrator/bootstrap.go index bfff2c3f65..3228610e76 100644 --- a/zetaclient/orchestrator/bootstrap.go +++ b/zetaclient/orchestrator/bootstrap.go @@ -334,7 +334,6 @@ func syncObserverMap( *params, client, tss, - cfg.RPCAlertLatency, database, logger, ts, @@ -370,7 +369,6 @@ func syncObserverMap( *params, client, tss, - cfg.RPCAlertLatency, database, logger, ts, @@ -407,7 +405,6 @@ func syncObserverMap( *params, client, tss, - cfg.RPCAlertLatency, database, logger, ts, @@ -437,7 +434,6 @@ func syncObserverMap( client, tss, base.DefaultBlockCacheSize, - cfg.RPCAlertLatency, ts, database, logger,