polygon/sync: milestone headers verifier (#10466)

Fixes:
```
[DBUG] [05-24|00:54:02.436] [sync] issue downloading waypoint blocks - will try again err="VerifyAccumulatedHeaders: bad headers root hash" start=7412843 end=7412895 rootHash=0x78a48a8d01ef08ba73edeef11f8c6b85aace76723ea53301a5732f69b21beee3 kind=*heimdall.Milestone peerId=c3a4ece91b544f27f2b01822bcdac0afd66df311abf8f76d6d23cba8d5ee1670f711312283b5636adda83f514b7f5badd499dc652a840d8abbb5c7649ae98ef1
```

Waypoint root hash verification is different for checkpoints and
milestones:
- checkpoints use merkle root hash of all the headers
- milestones use the last header hash

eth/stagedsync/stage_polygon_sync.go

@@ -52,14 +52,16 @@ func NewPolygonSyncStageCfg(
 		dataStream:  dataStream,
 	}
 	p2pService := p2p.NewService(maxPeers, logger, sentry, statusDataProvider.GetStatusData)
-	headersVerifier := polygonsync.VerifyAccumulatedHeaders
+	checkpointVerifier := polygonsync.VerifyCheckpointHeaders
+	milestoneVerifier := polygonsync.VerifyMilestoneHeaders
 	blocksVerifier := polygonsync.VerifyBlocks
 	heimdallService := heimdall.NewHeimdall(heimdallClient, logger, heimdall.WithStore(storage))
 	blockDownloader := polygonsync.NewBlockDownloader(
 		logger,
 		p2pService,
 		heimdallService,
-		headersVerifier,
+		checkpointVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		storage,
 	)
@@ -69,7 +71,7 @@ func NewPolygonSyncStageCfg(
 	sync := polygonsync.NewSync(
 		storage,
 		executionEngine,
-		headersVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		p2pService,
 		blockDownloader,

polygon/sync/block_downloader.go

@@ -37,15 +37,17 @@ func NewBlockDownloader(
 	logger log.Logger,
 	p2pService p2p.Service,
 	heimdall heimdall.Heimdall,
-	headersVerifier AccumulatedHeadersVerifier,
+	checkpointVerifier WaypointHeadersVerifier,
+	milestoneVerifier WaypointHeadersVerifier,
 	blocksVerifier BlocksVerifier,
 	store Store,
 ) BlockDownloader {
 	return newBlockDownloader(
 		logger,
 		p2pService,
 		heimdall,
-		headersVerifier,
+		checkpointVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		store,
 		notEnoughPeersBackOffDuration,
@@ -57,7 +59,8 @@ func newBlockDownloader(
 	logger log.Logger,
 	p2pService p2p.Service,
 	heimdall heimdall.Heimdall,
-	headersVerifier AccumulatedHeadersVerifier,
+	checkpointVerifier WaypointHeadersVerifier,
+	milestoneVerifier WaypointHeadersVerifier,
 	blocksVerifier BlocksVerifier,
 	store Store,
 	notEnoughPeersBackOffDuration time.Duration,
@@ -67,7 +70,8 @@ func newBlockDownloader(
 		logger:                        logger,
 		p2pService:                    p2pService,
 		heimdall:                      heimdall,
-		headersVerifier:               headersVerifier,
+		checkpointVerifier:            checkpointVerifier,
+		milestoneVerifier:             milestoneVerifier,
 		blocksVerifier:                blocksVerifier,
 		store:                         store,
 		notEnoughPeersBackOffDuration: notEnoughPeersBackOffDuration,
@@ -79,7 +83,8 @@ type blockDownloader struct {
 	logger                        log.Logger
 	p2pService                    p2p.Service
 	heimdall                      heimdall.Heimdall
-	headersVerifier               AccumulatedHeadersVerifier
+	checkpointVerifier            WaypointHeadersVerifier
+	milestoneVerifier             WaypointHeadersVerifier
 	blocksVerifier                BlocksVerifier
 	store                         Store
 	notEnoughPeersBackOffDuration time.Duration
@@ -92,7 +97,7 @@ func (d *blockDownloader) DownloadBlocksUsingCheckpoints(ctx context.Context, st
 		return nil, err
 	}
 
-	return d.downloadBlocksUsingWaypoints(ctx, waypoints)
+	return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.checkpointVerifier)
 }
 
 func (d *blockDownloader) DownloadBlocksUsingMilestones(ctx context.Context, start uint64) (*types.Header, error) {
@@ -101,10 +106,14 @@ func (d *blockDownloader) DownloadBlocksUsingMilestones(ctx context.Context, sta
 		return nil, err
 	}
 
-	return d.downloadBlocksUsingWaypoints(ctx, waypoints)
+	return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.milestoneVerifier)
 }
 
-func (d *blockDownloader) downloadBlocksUsingWaypoints(ctx context.Context, waypoints heimdall.Waypoints) (*types.Header, error) {
+func (d *blockDownloader) downloadBlocksUsingWaypoints(
+	ctx context.Context,
+	waypoints heimdall.Waypoints,
+	verifier WaypointHeadersVerifier,
+) (*types.Header, error) {
 	if len(waypoints) == 0 {
 		return nil, nil
 	}
@@ -192,7 +201,7 @@ func (d *blockDownloader) downloadBlocksUsingWaypoints(ctx context.Context, wayp
 					return
 				}
 
-				blocks, totalSize, err := d.fetchVerifiedBlocks(ctx, waypoint, peerId)
+				blocks, totalSize, err := d.fetchVerifiedBlocks(ctx, waypoint, peerId, verifier)
 				if err != nil {
 					d.logger.Debug(
 						syncLogPrefix("issue downloading waypoint blocks - will try again"),
@@ -270,6 +279,7 @@ func (d *blockDownloader) fetchVerifiedBlocks(
 	ctx context.Context,
 	waypoint heimdall.Waypoint,
 	peerId *p2p.PeerId,
+	verifier WaypointHeadersVerifier,
 ) ([]*types.Block, int, error) {
 	// 1. Fetch headers in waypoint from a peer
 	start := waypoint.StartBlock().Uint64()
@@ -280,7 +290,7 @@ func (d *blockDownloader) fetchVerifiedBlocks(
 	}
 
 	// 2. Verify headers match waypoint root hash
-	if err = d.headersVerifier(waypoint, headers.Data); err != nil {
+	if err = verifier(waypoint, headers.Data); err != nil {
 		d.logger.Debug(syncLogPrefix("penalizing peer - invalid headers"), "peerId", peerId, "err", err)
 
 		if penalizeErr := d.p2pService.Penalize(ctx, peerId); penalizeErr != nil {

polygon/sync/block_downloader_test.go

@@ -31,14 +31,16 @@ func newBlockDownloaderTestWithOpts(t *testing.T, opts blockDownloaderTestOpts)
 	p2pService := p2p.NewMockService(ctrl)
 	p2pService.EXPECT().MaxPeers().Return(100).Times(1)
 	logger := testlog.Logger(t, log.LvlDebug)
-	headersVerifier := opts.getOrCreateDefaultHeadersVerifier()
+	checkpointVerifier := opts.getOrCreateDefaultCheckpointVerifier()
+	milestoneVerifier := opts.getOrCreateDefaultMilestoneVerifier()
 	blocksVerifier := opts.getOrCreateDefaultBlocksVerifier()
 	store := NewMockStore(ctrl)
 	headerDownloader := newBlockDownloader(
 		logger,
 		p2pService,
 		heimdallService,
-		headersVerifier,
+		checkpointVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		store,
 		time.Millisecond,
@@ -53,19 +55,30 @@ func newBlockDownloaderTestWithOpts(t *testing.T, opts blockDownloaderTestOpts)
 }
 
 type blockDownloaderTestOpts struct {
-	headersVerifier AccumulatedHeadersVerifier
-	blocksVerifier  BlocksVerifier
-	maxWorkers      int
+	checkpointVerifier WaypointHeadersVerifier
+	milestoneVerifier  WaypointHeadersVerifier
+	blocksVerifier     BlocksVerifier
+	maxWorkers         int
 }
 
-func (opts blockDownloaderTestOpts) getOrCreateDefaultHeadersVerifier() AccumulatedHeadersVerifier {
-	if opts.headersVerifier == nil {
+func (opts blockDownloaderTestOpts) getOrCreateDefaultCheckpointVerifier() WaypointHeadersVerifier {
+	if opts.checkpointVerifier == nil {
 		return func(_ heimdall.Waypoint, _ []*types.Header) error {
 			return nil
 		}
 	}
 
-	return opts.headersVerifier
+	return opts.checkpointVerifier
+}
+
+func (opts blockDownloaderTestOpts) getOrCreateDefaultMilestoneVerifier() WaypointHeadersVerifier {
+	if opts.milestoneVerifier == nil {
+		return func(_ heimdall.Waypoint, _ []*types.Header) error {
+			return nil
+		}
+	}
+
+	return opts.milestoneVerifier
 }
 
 func (opts blockDownloaderTestOpts) getOrCreateDefaultBlocksVerifier() BlocksVerifier {
@@ -273,7 +286,7 @@ func TestBlockDownloaderDownloadBlocksWhenInvalidHeadersThenPenalizePeerAndReDow
 	var firstTimeInvalidReturned bool
 	firstTimeInvalidReturnedPtr := &firstTimeInvalidReturned
 	test := newBlockDownloaderTestWithOpts(t, blockDownloaderTestOpts{
-		headersVerifier: func(waypoint heimdall.Waypoint, headers []*types.Header) error {
+		checkpointVerifier: func(waypoint heimdall.Waypoint, headers []*types.Header) error {
 			if waypoint.StartBlock().Cmp(new(big.Int).SetUint64(2)) == 0 && !*firstTimeInvalidReturnedPtr {
 				*firstTimeInvalidReturnedPtr = true
 				return errors.New("invalid checkpoint")

polygon/sync/service.go

@@ -43,7 +43,8 @@ func NewService(
 	borConfig := chainConfig.Bor.(*borcfg.BorConfig)
 	execution := NewExecutionClient(executionClient)
 	store := NewStore(logger, execution)
-	headersVerifier := VerifyAccumulatedHeaders
+	checkpointVerifier := VerifyCheckpointHeaders
+	milestoneVerifier := VerifyMilestoneHeaders
 	blocksVerifier := VerifyBlocks
 	p2pService := p2p.NewService(maxPeers, logger, sentryClient, statusDataProvider.GetStatusData)
 	heimdallClient := heimdall.NewHeimdallClient(heimdallUrl, logger)
@@ -58,7 +59,8 @@ func NewService(
 		logger,
 		p2pService,
 		heimdallService,
-		headersVerifier,
+		checkpointVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		store,
 	)
@@ -68,7 +70,7 @@ func NewService(
 	sync := NewSync(
 		store,
 		execution,
-		headersVerifier,
+		milestoneVerifier,
 		blocksVerifier,
 		p2pService,
 		blockDownloader,

polygon/sync/sync.go

@@ -12,23 +12,23 @@ import (
 )
 
 type Sync struct {
-	store            Store
-	execution        ExecutionClient
-	headersVerifier  AccumulatedHeadersVerifier
-	blocksVerifier   BlocksVerifier
-	p2pService       p2p.Service
-	blockDownloader  BlockDownloader
-	ccBuilderFactory func(root *types.Header, span *heimdall.Span) CanonicalChainBuilder
-	spansCache       *SpansCache
-	fetchLatestSpan  func(ctx context.Context) (*heimdall.Span, error)
-	events           <-chan Event
-	logger           log.Logger
+	store             Store
+	execution         ExecutionClient
+	milestoneVerifier WaypointHeadersVerifier
+	blocksVerifier    BlocksVerifier
+	p2pService        p2p.Service
+	blockDownloader   BlockDownloader
+	ccBuilderFactory  func(root *types.Header, span *heimdall.Span) CanonicalChainBuilder
+	spansCache        *SpansCache
+	fetchLatestSpan   func(ctx context.Context) (*heimdall.Span, error)
+	events            <-chan Event
+	logger            log.Logger
 }
 
 func NewSync(
 	store Store,
 	execution ExecutionClient,
-	headersVerifier AccumulatedHeadersVerifier,
+	milestoneVerifier WaypointHeadersVerifier,
 	blocksVerifier BlocksVerifier,
 	p2pService p2p.Service,
 	blockDownloader BlockDownloader,
@@ -39,17 +39,17 @@ func NewSync(
 	logger log.Logger,
 ) *Sync {
 	return &Sync{
-		store:            store,
-		execution:        execution,
-		headersVerifier:  headersVerifier,
-		blocksVerifier:   blocksVerifier,
-		p2pService:       p2pService,
-		blockDownloader:  blockDownloader,
-		ccBuilderFactory: ccBuilderFactory,
-		spansCache:       spansCache,
-		fetchLatestSpan:  fetchLatestSpan,
-		events:           events,
-		logger:           logger,
+		store:             store,
+		execution:         execution,
+		milestoneVerifier: milestoneVerifier,
+		blocksVerifier:    blocksVerifier,
+		p2pService:        p2pService,
+		blockDownloader:   blockDownloader,
+		ccBuilderFactory:  ccBuilderFactory,
+		spansCache:        spansCache,
+		fetchLatestSpan:   fetchLatestSpan,
+		events:            events,
+		logger:            logger,
 	}
 }
 
@@ -71,7 +71,7 @@ func (s *Sync) onMilestoneEvent(
 	}
 
 	milestoneHeaders := ccBuilder.HeadersInRange(milestone.StartBlock().Uint64(), milestone.Length())
-	err := s.headersVerifier(milestone, milestoneHeaders)
+	err := s.milestoneVerifier(milestone, milestoneHeaders)
 	if err == nil {
 		if err = ccBuilder.Prune(milestone.EndBlock().Uint64()); err != nil {
 			return err

polygon/sync/waypoint_headers_verifier.go

@@ -0,0 +1,41 @@
+package sync
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/polygon/bor"
+	"github.com/ledgerwatch/erigon/polygon/heimdall"
+)
+
+var (
+	ErrFailedToComputeHeadersRootHash = errors.New("failed to compute headers root hash")
+	ErrBadHeadersRootHash             = errors.New("bad headers root hash")
+)
+
+type WaypointHeadersVerifier func(waypoint heimdall.Waypoint, headers []*types.Header) error
+
+func VerifyCheckpointHeaders(waypoint heimdall.Waypoint, headers []*types.Header) error {
+	rootHash, err := bor.ComputeHeadersRootHash(headers)
+	if err != nil {
+		return fmt.Errorf("VerifyCheckpointHeaders: %w: %w", ErrFailedToComputeHeadersRootHash, err)
+	}
+	if !bytes.Equal(rootHash, waypoint.RootHash().Bytes()) {
+		return fmt.Errorf("VerifyCheckpointHeaders: %w", ErrBadHeadersRootHash)
+	}
+	return nil
+}
+
+func VerifyMilestoneHeaders(waypoint heimdall.Waypoint, headers []*types.Header) error {
+	var hash common.Hash
+	if len(headers) > 0 {
+		hash = headers[len(headers)-1].Hash()
+	}
+	if hash != waypoint.RootHash() {
+		return fmt.Errorf("VerifyMilestoneHeaders: %w", ErrBadHeadersRootHash)
+	}
+	return nil
+}