diff --git a/module/builder/collection/builder_pebble.go b/module/builder/collection/builder_pebble.go new file mode 100644 index 00000000000..91f7fe93e37 --- /dev/null +++ b/module/builder/collection/builder_pebble.go @@ -0,0 +1,526 @@ +package collection + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/trace" + clusterstate "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/fork" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/logging" +) + +// Builder is the builder for collection block payloads. Upon providing a +// payload hash, it also memorizes the payload contents. +// +// NOTE: Builder is NOT safe for use with multiple goroutines. Since the +// HotStuff event loop is the only consumer of this interface and is single +// threaded, this is OK. +type Builder struct { + db *badger.DB + mainHeaders storage.Headers + clusterHeaders storage.Headers + protoState protocol.State + clusterState clusterstate.State + payloads storage.ClusterPayloads + transactions mempool.Transactions + tracer module.Tracer + config Config + log zerolog.Logger + clusterEpoch uint64 // the operating epoch for this cluster + // cache of values about the operating epoch which never change + refEpochFirstHeight uint64 // first height of this cluster's operating epoch + epochFinalHeight *uint64 // last height of this cluster's operating epoch (nil if epoch not ended) + epochFinalID *flow.Identifier // ID of last block in this cluster's operating epoch (nil if epoch not ended) +} + +func NewBuilder( + db *badger.DB, + tracer module.Tracer, + protoState protocol.State, + clusterState clusterstate.State, + mainHeaders storage.Headers, + clusterHeaders storage.Headers, + payloads storage.ClusterPayloads, + transactions mempool.Transactions, + log zerolog.Logger, + epochCounter uint64, + opts ...Opt, +) (*Builder, error) { + b := Builder{ + db: db, + tracer: tracer, + protoState: protoState, + clusterState: clusterState, + mainHeaders: mainHeaders, + clusterHeaders: clusterHeaders, + payloads: payloads, + transactions: transactions, + config: DefaultConfig(), + log: log.With().Str("component", "cluster_builder").Logger(), + clusterEpoch: epochCounter, + } + + err := db.View(operation.RetrieveEpochFirstHeight(epochCounter, &b.refEpochFirstHeight)) + if err != nil { + return nil, fmt.Errorf("could not get epoch first height: %w", err) + } + + for _, apply := range opts { + apply(&b.config) + } + + // sanity check config + if b.config.ExpiryBuffer >= flow.DefaultTransactionExpiry { + return nil, fmt.Errorf("invalid configured expiry buffer exceeds tx expiry (%d > %d)", b.config.ExpiryBuffer, flow.DefaultTransactionExpiry) + } + + return &b, nil +} + +// BuildOn creates a new block built on the given parent. It produces a payload +// that is valid with respect to the un-finalized chain it extends. +func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { + parentSpan, ctx := b.tracer.StartSpanFromContext(context.Background(), trace.COLBuildOn) + defer parentSpan.End() + + // STEP 1: build a lookup for excluding duplicated transactions. + // This is briefly how it works: + // + // Let E be the global transaction expiry. + // When incorporating a new collection C, with reference height R, we enforce + // that it contains only transactions with reference heights in [R,R+E). + // * if we are building C: + // * we don't build expired collections (ie. our local finalized consensus height is at most R+E-1) + // * we don't include transactions referencing un-finalized blocks + // * therefore, C will contain only transactions with reference heights in [R,R+E) + // * if we are validating C: + // * honest validators only consider C valid if all its transactions have reference heights in [R,R+E) + // + // Therefore, to check for duplicates, we only need a lookup for transactions in collection + // with expiry windows that overlap with our collection under construction. + // + // A collection with overlapping expiry window can be finalized or un-finalized. + // * to find all non-expired and finalized collections, we make use of an index + // (main_chain_finalized_height -> cluster_block_ids with respective reference height), to search for a range of main chain heights + // which could be only referenced by collections with overlapping expiry windows. + // * to find all overlapping and un-finalized collections, we can't use the above index, because it's + // only for finalized collections. Instead, we simply traverse along the chain up to the last + // finalized block. This could possibly include some collections with expiry windows that DON'T + // overlap with our collection under construction, but it is unlikely and doesn't impact correctness. + // + // After combining both the finalized and un-finalized cluster blocks that overlap with our expiry window, + // we can iterate through their transactions, and build a lookup for excluding duplicated transactions. + // + // RATE LIMITING: the builder module can be configured to limit the + // rate at which transactions with a common payer are included in + // blocks. Depending on the configured limit, we either allow 1 + // transaction every N sequential collections, or we allow K transactions + // per collection. The rate limiter tracks transactions included previously + // to enforce rate limit rules for the constructed block. + + span, _ := b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnGetBuildCtx) + buildCtx, err := b.getBlockBuildContext(parentID) + span.End() + if err != nil { + return nil, fmt.Errorf("could not get block build context: %w", err) + } + + log := b.log.With(). + Hex("parent_id", parentID[:]). + Str("chain_id", buildCtx.parent.ChainID.String()). + Uint64("final_ref_height", buildCtx.refChainFinalizedHeight). + Logger() + log.Debug().Msg("building new cluster block") + + // STEP 1a: create a lookup of all transactions included in UN-FINALIZED ancestors. + // In contrast to the transactions collected in step 1b, transactions in un-finalized + // collections cannot be removed from the mempool, as we would want to include + // such transactions in other forks. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnUnfinalizedLookup) + err = b.populateUnfinalizedAncestryLookup(buildCtx) + span.End() + if err != nil { + return nil, fmt.Errorf("could not populate un-finalized ancestry lookout (parent_id=%x): %w", parentID, err) + } + + // STEP 1b: create a lookup of all transactions previously included in + // the finalized collections. Any transactions already included in finalized + // collections can be removed from the mempool. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnFinalizedLookup) + err = b.populateFinalizedAncestryLookup(buildCtx) + span.End() + if err != nil { + return nil, fmt.Errorf("could not populate finalized ancestry lookup: %w", err) + } + + // STEP 2: build a payload of valid transactions, while at the same + // time figuring out the correct reference block ID for the collection. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnCreatePayload) + payload, err := b.buildPayload(buildCtx) + span.End() + if err != nil { + return nil, fmt.Errorf("could not build payload: %w", err) + } + + // STEP 3: we have a set of transactions that are valid to include on this fork. + // Now we create the header for the cluster block. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnCreateHeader) + header, err := b.buildHeader(buildCtx, payload, setter) + span.End() + if err != nil { + return nil, fmt.Errorf("could not build header: %w", err) + } + + proposal := cluster.Block{ + Header: header, + Payload: payload, + } + + // STEP 4: insert the cluster block to the database. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnDBInsert) + err = operation.RetryOnConflict(b.db.Update, procedure.InsertClusterBlock(&proposal)) + span.End() + if err != nil { + return nil, fmt.Errorf("could not insert built block: %w", err) + } + + return proposal.Header, nil +} + +// getBlockBuildContext retrieves the required contextual information from the database +// required to build a new block proposal. +// No errors are expected during normal operation. +func (b *Builder) getBlockBuildContext(parentID flow.Identifier) (*blockBuildContext, error) { + ctx := new(blockBuildContext) + ctx.config = b.config + ctx.parentID = parentID + ctx.lookup = newTransactionLookup() + + var err error + ctx.parent, err = b.clusterHeaders.ByBlockID(parentID) + if err != nil { + return nil, fmt.Errorf("could not get parent: %w", err) + } + ctx.limiter = newRateLimiter(b.config, ctx.parent.Height+1) + + // retrieve the finalized boundary ON THE CLUSTER CHAIN + ctx.clusterChainFinalizedBlock, err = b.clusterState.Final().Head() + if err != nil { + return nil, fmt.Errorf("could not retrieve cluster chain finalized header: %w", err) + } + + // retrieve the height and ID of the latest finalized block ON THE MAIN CHAIN + // this is used as the reference point for transaction expiry + mainChainFinalizedHeader, err := b.protoState.Final().Head() + if err != nil { + return nil, fmt.Errorf("could not retrieve main chain finalized header: %w", err) + } + ctx.refChainFinalizedHeight = mainChainFinalizedHeader.Height + ctx.refChainFinalizedID = mainChainFinalizedHeader.ID() + + // if the epoch has ended and the final block is cached, use the cached values + if b.epochFinalHeight != nil && b.epochFinalID != nil { + ctx.refEpochFinalID = b.epochFinalID + ctx.refEpochFinalHeight = b.epochFinalHeight + return ctx, nil + } + + // otherwise, attempt to read them from storage + err = b.db.View(func(btx *badger.Txn) error { + var refEpochFinalHeight uint64 + var refEpochFinalID flow.Identifier + + err = operation.RetrieveEpochLastHeight(b.clusterEpoch, &refEpochFinalHeight)(btx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil + } + return fmt.Errorf("unexpected failure to retrieve final height of operating epoch: %w", err) + } + err = operation.LookupBlockHeight(refEpochFinalHeight, &refEpochFinalID)(btx) + if err != nil { + // if we are able to retrieve the epoch's final height, the block must be finalized + // therefore failing to look up its height here is an unexpected error + return irrecoverable.NewExceptionf("could not retrieve ID of finalized final block of operating epoch: %w", err) + } + + // cache the values + b.epochFinalHeight = &refEpochFinalHeight + b.epochFinalID = &refEpochFinalID + // store the values in the build context + ctx.refEpochFinalID = b.epochFinalID + ctx.refEpochFinalHeight = b.epochFinalHeight + + return nil + }) + if err != nil { + return nil, fmt.Errorf("could not get block build context: %w", err) + } + return ctx, nil +} + +// populateUnfinalizedAncestryLookup traverses the unfinalized ancestry backward +// to populate the transaction lookup (used for deduplication) and the rate limiter +// (used to limit transaction submission by payer). +// +// The traversal begins with the block specified by parentID (the block we are +// building on top of) and ends with the oldest unfinalized block in the ancestry. +func (b *Builder) populateUnfinalizedAncestryLookup(ctx *blockBuildContext) error { + err := fork.TraverseBackward(b.clusterHeaders, ctx.parentID, func(ancestor *flow.Header) error { + payload, err := b.payloads.ByBlockID(ancestor.ID()) + if err != nil { + return fmt.Errorf("could not retrieve ancestor payload: %w", err) + } + + for _, tx := range payload.Collection.Transactions { + ctx.lookup.addUnfinalizedAncestor(tx.ID()) + ctx.limiter.addAncestor(ancestor.Height, tx) + } + return nil + }, fork.ExcludingHeight(ctx.clusterChainFinalizedBlock.Height)) + return err +} + +// populateFinalizedAncestryLookup traverses the reference block height index to +// populate the transaction lookup (used for deduplication) and the rate limiter +// (used to limit transaction submission by payer). +// +// The traversal is structured so that we check every collection whose reference +// block height translates to a possible constituent transaction which could also +// appear in the collection we are building. +func (b *Builder) populateFinalizedAncestryLookup(ctx *blockBuildContext) error { + minRefHeight := ctx.lowestPossibleReferenceBlockHeight() + maxRefHeight := ctx.highestPossibleReferenceBlockHeight() + lookup := ctx.lookup + limiter := ctx.limiter + + // Let E be the global transaction expiry constant, measured in blocks. For each + // T ∈ `includedTransactions`, we have to decide whether the transaction + // already appeared in _any_ finalized cluster block. + // Notation: + // - consider a valid cluster block C and let c be its reference block height + // - consider a transaction T ∈ `includedTransactions` and let t denote its + // reference block height + // + // Boundary conditions: + // 1. C's reference block height is equal to the lowest reference block height of + // all its constituent transactions. Hence, for collection C to potentially contain T, it must satisfy c <= t. + // 2. For T to be eligible for inclusion in collection C, _none_ of the transactions within C are allowed + // to be expired w.r.t. C's reference block. Hence, for collection C to potentially contain T, it must satisfy t < c + E. + // + // Therefore, for collection C to potentially contain transaction T, it must satisfy t - E < c <= t. + // In other words, we only need to inspect collections with reference block height c ∈ (t-E, t]. + // Consequently, for a set of transactions, with `minRefHeight` (`maxRefHeight`) being the smallest (largest) + // reference block height, we only need to inspect collections with c ∈ (minRefHeight-E, maxRefHeight]. + + // the finalized cluster blocks which could possibly contain any conflicting transactions + var clusterBlockIDs []flow.Identifier + start, end := findRefHeightSearchRangeForConflictingClusterBlocks(minRefHeight, maxRefHeight) + err := b.db.View(operation.LookupClusterBlocksByReferenceHeightRange(start, end, &clusterBlockIDs)) + if err != nil { + return fmt.Errorf("could not lookup finalized cluster blocks by reference height range [%d,%d]: %w", start, end, err) + } + + for _, blockID := range clusterBlockIDs { + header, err := b.clusterHeaders.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve cluster header (id=%x): %w", blockID, err) + } + payload, err := b.payloads.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve cluster payload (block_id=%x): %w", blockID, err) + } + for _, tx := range payload.Collection.Transactions { + lookup.addFinalizedAncestor(tx.ID()) + limiter.addAncestor(header.Height, tx) + } + } + + return nil +} + +// buildPayload constructs a valid payload based on transactions available in the mempool. +// If the mempool is empty, an empty payload will be returned. +// No errors are expected during normal operation. +func (b *Builder) buildPayload(buildCtx *blockBuildContext) (*cluster.Payload, error) { + lookup := buildCtx.lookup + limiter := buildCtx.limiter + maxRefHeight := buildCtx.highestPossibleReferenceBlockHeight() + // keep track of the actual smallest reference height of all included transactions + minRefHeight := maxRefHeight + minRefID := buildCtx.highestPossibleReferenceBlockID() + + var transactions []*flow.TransactionBody + var totalByteSize uint64 + var totalGas uint64 + for _, tx := range b.transactions.All() { + + // if we have reached maximum number of transactions, stop + if uint(len(transactions)) >= b.config.MaxCollectionSize { + break + } + + txByteSize := uint64(tx.ByteSize()) + // ignore transactions with tx byte size bigger that the max amount per collection + // this case shouldn't happen ever since we keep a limit on tx byte size but in case + // we keep this condition + if txByteSize > b.config.MaxCollectionByteSize { + continue + } + + // because the max byte size per tx is way smaller than the max collection byte size, we can stop here and not continue. + // to make it more effective in the future we can continue adding smaller ones + if totalByteSize+txByteSize > b.config.MaxCollectionByteSize { + break + } + + // ignore transactions with max gas bigger that the max total gas per collection + // this case shouldn't happen ever but in case we keep this condition + if tx.GasLimit > b.config.MaxCollectionTotalGas { + continue + } + + // cause the max gas limit per tx is way smaller than the total max gas per collection, we can stop here and not continue. + // to make it more effective in the future we can continue adding smaller ones + if totalGas+tx.GasLimit > b.config.MaxCollectionTotalGas { + break + } + + // retrieve the main chain header that was used as reference + refHeader, err := b.mainHeaders.ByBlockID(tx.ReferenceBlockID) + if errors.Is(err, storage.ErrNotFound) { + continue // in case we are configured with liberal transaction ingest rules + } + if err != nil { + return nil, fmt.Errorf("could not retrieve reference header: %w", err) + } + + // disallow un-finalized reference blocks, and reference blocks beyond the cluster's operating epoch + if refHeader.Height > maxRefHeight { + continue + } + + txID := tx.ID() + // make sure the reference block is finalized and not orphaned + blockIDFinalizedAtRefHeight, err := b.mainHeaders.BlockIDByHeight(refHeader.Height) + if err != nil { + return nil, fmt.Errorf("could not check that reference block (id=%x) for transaction (id=%x) is finalized: %w", tx.ReferenceBlockID, txID, err) + } + if blockIDFinalizedAtRefHeight != tx.ReferenceBlockID { + // the transaction references an orphaned block - it will never be valid + b.transactions.Remove(txID) + continue + } + + // ensure the reference block is not too old + if refHeader.Height < buildCtx.lowestPossibleReferenceBlockHeight() { + // the transaction is expired, it will never be valid + b.transactions.Remove(txID) + continue + } + + // check that the transaction was not already used in un-finalized history + if lookup.isUnfinalizedAncestor(txID) { + continue + } + + // check that the transaction was not already included in finalized history. + if lookup.isFinalizedAncestor(txID) { + // remove from mempool, conflicts with finalized block will never be valid + b.transactions.Remove(txID) + continue + } + + // enforce rate limiting rules + if limiter.shouldRateLimit(tx) { + if b.config.DryRunRateLimit { + // log that this transaction would have been rate-limited, but we will still include it in the collection + b.log.Info(). + Hex("tx_id", logging.ID(txID)). + Str("payer_addr", tx.Payer.String()). + Float64("rate_limit", b.config.MaxPayerTransactionRate). + Msg("dry-run: observed transaction that would have been rate limited") + } else { + b.log.Debug(). + Hex("tx_id", logging.ID(txID)). + Str("payer_addr", tx.Payer.String()). + Float64("rate_limit", b.config.MaxPayerTransactionRate). + Msg("transaction is rate-limited") + continue + } + } + + // ensure we find the lowest reference block height + if refHeader.Height < minRefHeight { + minRefHeight = refHeader.Height + minRefID = tx.ReferenceBlockID + } + + // update per-payer transaction count + limiter.transactionIncluded(tx) + + transactions = append(transactions, tx) + totalByteSize += txByteSize + totalGas += tx.GasLimit + } + + // build the payload from the transactions + payload := cluster.PayloadFromTransactions(minRefID, transactions...) + return &payload, nil +} + +// buildHeader constructs the header for the cluster block being built. +// It invokes the HotStuff setter to set fields related to HotStuff (QC, etc.). +// No errors are expected during normal operation. +func (b *Builder) buildHeader(ctx *blockBuildContext, payload *cluster.Payload, setter func(header *flow.Header) error) (*flow.Header, error) { + + header := &flow.Header{ + ChainID: ctx.parent.ChainID, + ParentID: ctx.parentID, + Height: ctx.parent.Height + 1, + PayloadHash: payload.Hash(), + Timestamp: time.Now().UTC(), + + // NOTE: we rely on the HotStuff-provided setter to set the other + // fields, which are related to signatures and HotStuff internals + } + + // set fields specific to the consensus algorithm + err := setter(header) + if err != nil { + return nil, fmt.Errorf("could not set fields to header: %w", err) + } + return header, nil +} + +// findRefHeightSearchRangeForConflictingClusterBlocks computes the range of reference +// block heights of ancestor blocks which could possibly contain transactions +// duplicating those in our collection under construction, based on the range of +// reference heights of transactions in the collection under construction. +// +// Input range is the (inclusive) range of reference heights of transactions included +// in the collection under construction. Output range is the (inclusive) range of +// reference heights which need to be searched. +func findRefHeightSearchRangeForConflictingClusterBlocks(minRefHeight, maxRefHeight uint64) (start, end uint64) { + start = minRefHeight - flow.DefaultTransactionExpiry + 1 + if start > minRefHeight { + start = 0 // overflow check + } + end = maxRefHeight + return start, end +} diff --git a/module/builder/collection/builder_pebble_test.go b/module/builder/collection/builder_pebble_test.go new file mode 100644 index 00000000000..9641b7c934a --- /dev/null +++ b/module/builder/collection/builder_pebble_test.go @@ -0,0 +1,1048 @@ +package collection_test + +import ( + "context" + "math/rand" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + model "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + builder "github.com/onflow/flow-go/module/builder/collection" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/cluster" + clusterkv "github.com/onflow/flow-go/state/cluster/badger" + "github.com/onflow/flow-go/state/protocol" + pbadger "github.com/onflow/flow-go/state/protocol/badger" + "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/util" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + sutil "github.com/onflow/flow-go/storage/util" + "github.com/onflow/flow-go/utils/unittest" +) + +var noopSetter = func(*flow.Header) error { return nil } + +type BuilderSuite struct { + suite.Suite + db *badger.DB + dbdir string + + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 + + headers storage.Headers + payloads storage.ClusterPayloads + blocks storage.Blocks + + state cluster.MutableState + + // protocol state for reference blocks for transactions + protoState protocol.FollowerState + + pool mempool.Transactions + builder *builder.Builder +} + +// runs before each test runs +func (suite *BuilderSuite) SetupTest() { + var err error + + suite.genesis = model.Genesis() + suite.chainID = suite.genesis.Header.ChainID + + suite.pool = herocache.NewTransactions(1000, unittest.Logger(), metrics.NewNoopCollector()) + + suite.dbdir = unittest.TempDir(suite.T()) + suite.db = unittest.BadgerDB(suite.T(), suite.dbdir) + + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := sutil.StorageLayer(suite.T(), suite.db) + consumer := events.NewNoop() + + suite.headers = all.Headers + suite.blocks = all.Blocks + suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) + + // just bootstrap with a genesis block, we'll use this as reference + root, result, seal := unittest.BootstrapFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + // ensure we don't enter a new epoch for tests that build many blocks + result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 + seal.ResultID = result.ID() + rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) + require.NoError(suite.T(), err) + suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter + + clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) + clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC, suite.epochCounter) + suite.Require().NoError(err) + clusterState, err := clusterkv.Bootstrap(suite.db, clusterStateRoot) + suite.Require().NoError(err) + + suite.state, err = clusterkv.NewMutableState(clusterState, tracer, suite.headers, suite.payloads) + suite.Require().NoError(err) + + state, err := pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(suite.T(), err) + + suite.protoState, err = pbadger.NewFollowerState( + log, + tracer, + consumer, + state, + all.Index, + all.Payloads, + util.MockBlockTimer(), + ) + require.NoError(suite.T(), err) + + // add some transactions to transaction pool + for i := 0; i < 3; i++ { + transaction := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { + tx.ReferenceBlockID = root.ID() + tx.ProposalKey.SequenceNumber = uint64(i) + tx.GasLimit = uint64(9999) + }) + added := suite.pool.Add(&transaction) + suite.Assert().True(added) + } + + suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) +} + +// runs after each test finishes +func (suite *BuilderSuite) TearDownTest() { + err := suite.db.Close() + suite.Assert().NoError(err) + err = os.RemoveAll(suite.dbdir) + suite.Assert().NoError(err) +} + +func (suite *BuilderSuite) InsertBlock(block model.Block) { + err := suite.db.Update(procedure.InsertClusterBlock(&block)) + suite.Assert().NoError(err) +} + +func (suite *BuilderSuite) FinalizeBlock(block model.Block) { + err := suite.db.Update(func(tx *badger.Txn) error { + var refBlock flow.Header + err := operation.RetrieveHeader(block.Payload.ReferenceBlockID, &refBlock)(tx) + if err != nil { + return err + } + err = procedure.FinalizeClusterBlock(block.ID())(tx) + if err != nil { + return err + } + err = operation.IndexClusterBlockByReferenceHeight(refBlock.Height, block.ID())(tx) + return err + }) + suite.Assert().NoError(err) +} + +// Payload returns a payload containing the given transactions, with a valid +// reference block ID. +func (suite *BuilderSuite) Payload(transactions ...*flow.TransactionBody) model.Payload { + final, err := suite.protoState.Final().Head() + suite.Require().NoError(err) + return model.PayloadFromTransactions(final.ID(), transactions...) +} + +// ProtoStateRoot returns the root block of the protocol state. +func (suite *BuilderSuite) ProtoStateRoot() *flow.Header { + root, err := suite.protoState.Params().FinalizedRoot() + suite.Require().NoError(err) + return root +} + +// ClearPool removes all items from the pool +func (suite *BuilderSuite) ClearPool() { + // TODO use Clear() + for _, tx := range suite.pool.All() { + suite.pool.Remove(tx.ID()) + } +} + +// FillPool adds n transactions to the pool, using the given generator function. +func (suite *BuilderSuite) FillPool(n int, create func() *flow.TransactionBody) { + for i := 0; i < n; i++ { + tx := create() + suite.pool.Add(tx) + } +} + +func TestBuilder(t *testing.T) { + suite.Run(t, new(BuilderSuite)) +} + +func (suite *BuilderSuite) TestBuildOn_NonExistentParent() { + // use a non-existent parent ID + parentID := unittest.IdentifierFixture() + + _, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Assert().Error(err) +} + +func (suite *BuilderSuite) TestBuildOn_Success() { + + var expectedHeight uint64 = 42 + setter := func(h *flow.Header) error { + h.Height = expectedHeight + return nil + } + + header, err := suite.builder.BuildOn(suite.genesis.ID(), setter) + suite.Require().NoError(err) + + // setter should have been run + suite.Assert().Equal(expectedHeight, header.Height) + + // should be able to retrieve built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + builtCollection := built.Payload.Collection + + // should reference a valid reference block + // (since genesis is the only block, it's the only valid reference) + mainGenesis, err := suite.protoState.AtHeight(0).Head() + suite.Assert().NoError(err) + suite.Assert().Equal(mainGenesis.ID(), built.Payload.ReferenceBlockID) + + // payload should include only items from mempool + mempoolTransactions := suite.pool.All() + suite.Assert().Len(builtCollection.Transactions, 3) + suite.Assert().True(collectionContains(builtCollection, flow.GetIDs(mempoolTransactions)...)) +} + +// when there are transactions with an unknown reference block in the pool, we should not include them in collections +func (suite *BuilderSuite) TestBuildOn_WithUnknownReferenceBlock() { + + // before modifying the mempool, note the valid transactions already in the pool + validMempoolTransactions := suite.pool.All() + + // add a transaction unknown reference block to the pool + unknownReferenceTx := unittest.TransactionBodyFixture() + unknownReferenceTx.ReferenceBlockID = unittest.IdentifierFixture() + suite.pool.Add(&unknownReferenceTx) + + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // should be able to retrieve built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + builtCollection := built.Payload.Collection + + suite.Assert().Len(builtCollection.Transactions, 3) + // payload should include only the transactions with a valid reference block + suite.Assert().True(collectionContains(builtCollection, flow.GetIDs(validMempoolTransactions)...)) + // should not contain the unknown-reference transaction + suite.Assert().False(collectionContains(builtCollection, unknownReferenceTx.ID())) +} + +// when there are transactions with a known but unfinalized reference block in the pool, we should not include them in collections +func (suite *BuilderSuite) TestBuildOn_WithUnfinalizedReferenceBlock() { + + // before modifying the mempool, note the valid transactions already in the pool + validMempoolTransactions := suite.pool.All() + + // add an unfinalized block to the protocol state + genesis, err := suite.protoState.Final().Head() + suite.Require().NoError(err) + unfinalizedReferenceBlock := unittest.BlockWithParentFixture(genesis) + unfinalizedReferenceBlock.SetPayload(flow.EmptyPayload()) + err = suite.protoState.ExtendCertified(context.Background(), unfinalizedReferenceBlock, + unittest.CertifyBlock(unfinalizedReferenceBlock.Header)) + suite.Require().NoError(err) + + // add a transaction with unfinalized reference block to the pool + unfinalizedReferenceTx := unittest.TransactionBodyFixture() + unfinalizedReferenceTx.ReferenceBlockID = unfinalizedReferenceBlock.ID() + suite.pool.Add(&unfinalizedReferenceTx) + + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // should be able to retrieve built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + builtCollection := built.Payload.Collection + + suite.Assert().Len(builtCollection.Transactions, 3) + // payload should include only the transactions with a valid reference block + suite.Assert().True(collectionContains(builtCollection, flow.GetIDs(validMempoolTransactions)...)) + // should not contain the unfinalized-reference transaction + suite.Assert().False(collectionContains(builtCollection, unfinalizedReferenceTx.ID())) +} + +// when there are transactions with an orphaned reference block in the pool, we should not include them in collections +func (suite *BuilderSuite) TestBuildOn_WithOrphanedReferenceBlock() { + + // before modifying the mempool, note the valid transactions already in the pool + validMempoolTransactions := suite.pool.All() + + // add an orphaned block to the protocol state + genesis, err := suite.protoState.Final().Head() + suite.Require().NoError(err) + // create a block extending genesis which will be orphaned + orphan := unittest.BlockWithParentFixture(genesis) + orphan.SetPayload(flow.EmptyPayload()) + err = suite.protoState.ExtendCertified(context.Background(), orphan, unittest.CertifyBlock(orphan.Header)) + suite.Require().NoError(err) + // create and finalize a block on top of genesis, orphaning `orphan` + block1 := unittest.BlockWithParentFixture(genesis) + block1.SetPayload(flow.EmptyPayload()) + err = suite.protoState.ExtendCertified(context.Background(), block1, unittest.CertifyBlock(block1.Header)) + suite.Require().NoError(err) + err = suite.protoState.Finalize(context.Background(), block1.ID()) + suite.Require().NoError(err) + + // add a transaction with orphaned reference block to the pool + orphanedReferenceTx := unittest.TransactionBodyFixture() + orphanedReferenceTx.ReferenceBlockID = orphan.ID() + suite.pool.Add(&orphanedReferenceTx) + + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // should be able to retrieve built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + builtCollection := built.Payload.Collection + + suite.Assert().Len(builtCollection.Transactions, 3) + // payload should include only the transactions with a valid reference block + suite.Assert().True(collectionContains(builtCollection, flow.GetIDs(validMempoolTransactions)...)) + // should not contain the unknown-reference transaction + suite.Assert().False(collectionContains(builtCollection, orphanedReferenceTx.ID())) + // the transaction with orphaned reference should be removed from the mempool + suite.Assert().False(suite.pool.Has(orphanedReferenceTx.ID())) +} + +func (suite *BuilderSuite) TestBuildOn_WithForks() { + t := suite.T() + + mempoolTransactions := suite.pool.All() + tx1 := mempoolTransactions[0] // in fork 1 + tx2 := mempoolTransactions[1] // in fork 2 + tx3 := mempoolTransactions[2] // in no block + + // build first fork on top of genesis + payload1 := suite.Payload(tx1) + block1 := unittest.ClusterBlockWithParent(suite.genesis) + block1.SetPayload(payload1) + + // insert block on fork 1 + suite.InsertBlock(block1) + + // build second fork on top of genesis + payload2 := suite.Payload(tx2) + block2 := unittest.ClusterBlockWithParent(suite.genesis) + block2.SetPayload(payload2) + + // insert block on fork 2 + suite.InsertBlock(block2) + + // build on top of fork 1 + header, err := suite.builder.BuildOn(block1.ID(), noopSetter) + require.NoError(t, err) + + // should be able to retrieve built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + assert.NoError(t, err) + builtCollection := built.Payload.Collection + + // payload should include ONLY tx2 and tx3 + assert.Len(t, builtCollection.Transactions, 2) + assert.True(t, collectionContains(builtCollection, tx2.ID(), tx3.ID())) + assert.False(t, collectionContains(builtCollection, tx1.ID())) +} + +func (suite *BuilderSuite) TestBuildOn_ConflictingFinalizedBlock() { + t := suite.T() + + mempoolTransactions := suite.pool.All() + tx1 := mempoolTransactions[0] // in a finalized block + tx2 := mempoolTransactions[1] // in an un-finalized block + tx3 := mempoolTransactions[2] // in no blocks + + t.Logf("tx1: %s\ntx2: %s\ntx3: %s", tx1.ID(), tx2.ID(), tx3.ID()) + + // build a block containing tx1 on genesis + finalizedPayload := suite.Payload(tx1) + finalizedBlock := unittest.ClusterBlockWithParent(suite.genesis) + finalizedBlock.SetPayload(finalizedPayload) + suite.InsertBlock(finalizedBlock) + t.Logf("finalized: height=%d id=%s txs=%s parent_id=%s\t\n", finalizedBlock.Header.Height, finalizedBlock.ID(), finalizedPayload.Collection.Light(), finalizedBlock.Header.ParentID) + + // build a block containing tx2 on the first block + unFinalizedPayload := suite.Payload(tx2) + unFinalizedBlock := unittest.ClusterBlockWithParent(&finalizedBlock) + unFinalizedBlock.SetPayload(unFinalizedPayload) + suite.InsertBlock(unFinalizedBlock) + t.Logf("finalized: height=%d id=%s txs=%s parent_id=%s\t\n", unFinalizedBlock.Header.Height, unFinalizedBlock.ID(), unFinalizedPayload.Collection.Light(), unFinalizedBlock.Header.ParentID) + + // finalize first block + suite.FinalizeBlock(finalizedBlock) + + // build on the un-finalized block + header, err := suite.builder.BuildOn(unFinalizedBlock.ID(), noopSetter) + require.NoError(t, err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + assert.NoError(t, err) + builtCollection := built.Payload.Collection + + // payload should only contain tx3 + assert.Len(t, builtCollection.Light().Transactions, 1) + assert.True(t, collectionContains(builtCollection, tx3.ID())) + assert.False(t, collectionContains(builtCollection, tx1.ID(), tx2.ID())) + + // tx1 should be removed from mempool, as it is in a finalized block + assert.False(t, suite.pool.Has(tx1.ID())) + // tx2 should NOT be removed from mempool, as it is in an un-finalized block + assert.True(t, suite.pool.Has(tx2.ID())) +} + +func (suite *BuilderSuite) TestBuildOn_ConflictingInvalidatedForks() { + t := suite.T() + + mempoolTransactions := suite.pool.All() + tx1 := mempoolTransactions[0] // in a finalized block + tx2 := mempoolTransactions[1] // in an invalidated block + tx3 := mempoolTransactions[2] // in no blocks + + t.Logf("tx1: %s\ntx2: %s\ntx3: %s", tx1.ID(), tx2.ID(), tx3.ID()) + + // build a block containing tx1 on genesis - will be finalized + finalizedPayload := suite.Payload(tx1) + finalizedBlock := unittest.ClusterBlockWithParent(suite.genesis) + finalizedBlock.SetPayload(finalizedPayload) + + suite.InsertBlock(finalizedBlock) + t.Logf("finalized: id=%s\tparent_id=%s\theight=%d\n", finalizedBlock.ID(), finalizedBlock.Header.ParentID, finalizedBlock.Header.Height) + + // build a block containing tx2 ALSO on genesis - will be invalidated + invalidatedPayload := suite.Payload(tx2) + invalidatedBlock := unittest.ClusterBlockWithParent(suite.genesis) + invalidatedBlock.SetPayload(invalidatedPayload) + suite.InsertBlock(invalidatedBlock) + t.Logf("invalidated: id=%s\tparent_id=%s\theight=%d\n", invalidatedBlock.ID(), invalidatedBlock.Header.ParentID, invalidatedBlock.Header.Height) + + // finalize first block - this indirectly invalidates the second block + suite.FinalizeBlock(finalizedBlock) + + // build on the finalized block + header, err := suite.builder.BuildOn(finalizedBlock.ID(), noopSetter) + require.NoError(t, err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + assert.NoError(t, err) + builtCollection := built.Payload.Collection + + // tx2 and tx3 should be in the built collection + assert.Len(t, builtCollection.Light().Transactions, 2) + assert.True(t, collectionContains(builtCollection, tx2.ID(), tx3.ID())) + assert.False(t, collectionContains(builtCollection, tx1.ID())) +} + +func (suite *BuilderSuite) TestBuildOn_LargeHistory() { + t := suite.T() + + // use a mempool with 2000 transactions, one per block + suite.pool = herocache.NewTransactions(2000, unittest.Logger(), metrics.NewNoopCollector()) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10000)) + + // get a valid reference block ID + final, err := suite.protoState.Final().Head() + require.NoError(t, err) + refID := final.ID() + + // keep track of the head of the chain + head := *suite.genesis + + // keep track of invalidated transaction IDs + var invalidatedTxIds []flow.Identifier + + // create a large history of blocks with invalidated forks every 3 blocks on + // average - build until the height exceeds transaction expiry + for i := 0; ; i++ { + + // create a transaction + tx := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { + tx.ReferenceBlockID = refID + tx.ProposalKey.SequenceNumber = uint64(i) + }) + added := suite.pool.Add(&tx) + assert.True(t, added) + + // 1/3 of the time create a conflicting fork that will be invalidated + // don't do this the first and last few times to ensure we don't + // try to fork genesis and the last block is the valid fork. + conflicting := rand.Intn(3) == 0 && i > 5 && i < 995 + + // by default, build on the head - if we are building a + // conflicting fork, build on the parent of the head + parent := head + if conflicting { + err = suite.db.View(procedure.RetrieveClusterBlock(parent.Header.ParentID, &parent)) + assert.NoError(t, err) + // add the transaction to the invalidated list + invalidatedTxIds = append(invalidatedTxIds, tx.ID()) + } + + // create a block containing the transaction + block := unittest.ClusterBlockWithParent(&head) + payload := suite.Payload(&tx) + block.SetPayload(payload) + suite.InsertBlock(block) + + // reset the valid head if we aren't building a conflicting fork + if !conflicting { + head = block + suite.FinalizeBlock(block) + assert.NoError(t, err) + } + + // stop building blocks once we've built a history which exceeds the transaction + // expiry length - this tests that deduplication works properly against old blocks + // which nevertheless have a potentially conflicting reference block + if head.Header.Height > flow.DefaultTransactionExpiry+100 { + break + } + } + + t.Log("conflicting: ", len(invalidatedTxIds)) + + // build on the head block + header, err := suite.builder.BuildOn(head.ID(), noopSetter) + require.NoError(t, err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + require.NoError(t, err) + builtCollection := built.Payload.Collection + + // payload should only contain transactions from invalidated blocks + assert.Len(t, builtCollection.Transactions, len(invalidatedTxIds), "expected len=%d, got len=%d", len(invalidatedTxIds), len(builtCollection.Transactions)) + assert.True(t, collectionContains(builtCollection, invalidatedTxIds...)) +} + +func (suite *BuilderSuite) TestBuildOn_MaxCollectionSize() { + // set the max collection size to 1 + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(1)) + + // build a block + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Require().NoError(err) + builtCollection := built.Payload.Collection + + // should be only 1 transaction in the collection + suite.Assert().Equal(builtCollection.Len(), 1) +} + +func (suite *BuilderSuite) TestBuildOn_MaxCollectionByteSize() { + // set the max collection byte size to 400 (each tx is about 150 bytes) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionByteSize(400)) + + // build a block + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Require().NoError(err) + builtCollection := built.Payload.Collection + + // should be only 2 transactions in the collection, since each tx is ~273 bytes and the limit is 600 bytes + suite.Assert().Equal(builtCollection.Len(), 2) +} + +func (suite *BuilderSuite) TestBuildOn_MaxCollectionTotalGas() { + // set the max gas to 20,000 + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionTotalGas(20000)) + + // build a block + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Require().NoError(err) + builtCollection := built.Payload.Collection + + // should be only 2 transactions in collection, since each transaction has gas limit of 9,999 and collection limit is set to 20,000 + suite.Assert().Equal(builtCollection.Len(), 2) +} + +func (suite *BuilderSuite) TestBuildOn_ExpiredTransaction() { + + // create enough main-chain blocks that an expired transaction is possible + genesis, err := suite.protoState.Final().Head() + suite.Require().NoError(err) + + head := genesis + for i := 0; i < flow.DefaultTransactionExpiry+1; i++ { + block := unittest.BlockWithParentFixture(head) + block.Payload.Guarantees = nil + block.Payload.Seals = nil + block.Header.PayloadHash = block.Payload.Hash() + err = suite.protoState.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + suite.Require().NoError(err) + err = suite.protoState.Finalize(context.Background(), block.ID()) + suite.Require().NoError(err) + head = block.Header + } + + // reset the pool and builder + suite.pool = herocache.NewTransactions(10, unittest.Logger(), metrics.NewNoopCollector()) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) + + // insert a transaction referring genesis (now expired) + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { + tx.ReferenceBlockID = genesis.ID() + tx.ProposalKey.SequenceNumber = 0 + }) + added := suite.pool.Add(&tx1) + suite.Assert().True(added) + + // insert a transaction referencing the head (valid) + tx2 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { + tx.ReferenceBlockID = head.ID() + tx.ProposalKey.SequenceNumber = 1 + }) + added = suite.pool.Add(&tx2) + suite.Assert().True(added) + + suite.T().Log("tx1: ", tx1.ID()) + suite.T().Log("tx2: ", tx2.ID()) + + // build a block + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + // retrieve the built block from storage + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Require().NoError(err) + builtCollection := built.Payload.Collection + + // the block should only contain the un-expired transaction + suite.Assert().False(collectionContains(builtCollection, tx1.ID())) + suite.Assert().True(collectionContains(builtCollection, tx2.ID())) + // the expired transaction should have been removed from the mempool + suite.Assert().False(suite.pool.Has(tx1.ID())) +} + +func (suite *BuilderSuite) TestBuildOn_EmptyMempool() { + + // start with an empty mempool + suite.pool = herocache.NewTransactions(1000, unittest.Logger(), metrics.NewNoopCollector()) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) + + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + suite.Require().NoError(err) + + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Require().NoError(err) + + // should reference a valid reference block + // (since genesis is the only block, it's the only valid reference) + mainGenesis, err := suite.protoState.AtHeight(0).Head() + suite.Assert().NoError(err) + suite.Assert().Equal(mainGenesis.ID(), built.Payload.ReferenceBlockID) + + // the payload should be empty + suite.Assert().Equal(0, built.Payload.Collection.Len()) +} + +// With rate limiting turned off, we should fill collections as fast as we can +// regardless of how many transactions with the same payer we include. +func (suite *BuilderSuite) TestBuildOn_NoRateLimiting() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with no rate limit and max 10 tx/collection + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(0), + ) + + // fill the pool with 100 transactions from the same payer + payer := unittest.RandomAddressFixture() + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = payer + return &tx + } + suite.FillPool(100, create) + + // since we have no rate limiting we should fill all collections and in 10 blocks + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // each collection should be full with 10 transactions + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + suite.Assert().Len(built.Payload.Collection.Transactions, 10) + } +} + +// With rate limiting turned on, we should be able to fill transactions as fast +// as possible so long as per-payer limits are not reached. This test generates +// transactions such that the number of transactions with a given proposer exceeds +// the rate limit -- since it's the proposer not the payer, it shouldn't limit +// our collections. +func (suite *BuilderSuite) TestBuildOn_RateLimitNonPayer() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with 5 tx/payer and max 10 tx/collection + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(5), + ) + + // fill the pool with 100 transactions with the same proposer + // since it's not the same payer, rate limit does not apply + proposer := unittest.RandomAddressFixture() + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = unittest.RandomAddressFixture() + tx.ProposalKey = flow.ProposalKey{ + Address: proposer, + KeyIndex: rand.Uint64(), + SequenceNumber: rand.Uint64(), + } + return &tx + } + suite.FillPool(100, create) + + // since rate limiting does not apply to non-payer keys, we should fill all collections in 10 blocks + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // each collection should be full with 10 transactions + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + suite.Assert().Len(built.Payload.Collection.Transactions, 10) + } +} + +// When configured with a rate limit of k>1, we should be able to include up to +// k transactions with a given payer per collection +func (suite *BuilderSuite) TestBuildOn_HighRateLimit() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with 5 tx/payer and max 10 tx/collection + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(5), + ) + + // fill the pool with 50 transactions from the same payer + payer := unittest.RandomAddressFixture() + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = payer + return &tx + } + suite.FillPool(50, create) + + // rate-limiting should be applied, resulting in half-full collections (5/10) + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // each collection should be half-full with 5 transactions + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + suite.Assert().Len(built.Payload.Collection.Transactions, 5) + } +} + +// When configured with a rate limit of k<1, we should be able to include 1 +// transactions with a given payer every ceil(1/k) collections +func (suite *BuilderSuite) TestBuildOn_LowRateLimit() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with .5 tx/payer and max 10 tx/collection + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(.5), + ) + + // fill the pool with 5 transactions from the same payer + payer := unittest.RandomAddressFixture() + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = payer + return &tx + } + suite.FillPool(5, create) + + // rate-limiting should be applied, resulting in every ceil(1/k) collections + // having one transaction and empty collections otherwise + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // collections should either be empty or have 1 transaction + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + if i%2 == 0 { + suite.Assert().Len(built.Payload.Collection.Transactions, 1) + } else { + suite.Assert().Len(built.Payload.Collection.Transactions, 0) + } + } +} +func (suite *BuilderSuite) TestBuildOn_UnlimitedPayer() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with 5 tx/payer and max 10 tx/collection + // configure an unlimited payer + payer := unittest.RandomAddressFixture() + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(5), + builder.WithUnlimitedPayers(payer), + ) + + // fill the pool with 100 transactions from the same payer + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = payer + return &tx + } + suite.FillPool(100, create) + + // rate-limiting should not be applied, since the payer is marked as unlimited + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // each collection should be full with 10 transactions + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + suite.Assert().Len(built.Payload.Collection.Transactions, 10) + + } +} + +// TestBuildOn_RateLimitDryRun tests that rate limiting rules aren't enforced +// if dry-run is enabled. +func (suite *BuilderSuite) TestBuildOn_RateLimitDryRun() { + + // start with an empty mempool + suite.ClearPool() + + // create builder with 5 tx/payer and max 10 tx/collection + // configure an unlimited payer + payer := unittest.RandomAddressFixture() + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, + builder.WithMaxCollectionSize(10), + builder.WithMaxPayerTransactionRate(5), + builder.WithRateLimitDryRun(true), + ) + + // fill the pool with 100 transactions from the same payer + create := func() *flow.TransactionBody { + tx := unittest.TransactionBodyFixture() + tx.ReferenceBlockID = suite.ProtoStateRoot().ID() + tx.Payer = payer + return &tx + } + suite.FillPool(100, create) + + // rate-limiting should not be applied, since dry-run setting is enabled + parentID := suite.genesis.ID() + for i := 0; i < 10; i++ { + header, err := suite.builder.BuildOn(parentID, noopSetter) + suite.Require().NoError(err) + parentID = header.ID() + + // each collection should be full with 10 transactions + var built model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(header.ID(), &built)) + suite.Assert().NoError(err) + suite.Assert().Len(built.Payload.Collection.Transactions, 10) + } +} + +// helper to check whether a collection contains each of the given transactions. +func collectionContains(collection flow.Collection, txIDs ...flow.Identifier) bool { + + lookup := make(map[flow.Identifier]struct{}, len(txIDs)) + for _, tx := range collection.Transactions { + lookup[tx.ID()] = struct{}{} + } + + for _, txID := range txIDs { + _, exists := lookup[txID] + if !exists { + return false + } + } + + return true +} + +func BenchmarkBuildOn10(b *testing.B) { benchmarkBuildOn(b, 10) } +func BenchmarkBuildOn100(b *testing.B) { benchmarkBuildOn(b, 100) } +func BenchmarkBuildOn1000(b *testing.B) { benchmarkBuildOn(b, 1000) } +func BenchmarkBuildOn10000(b *testing.B) { benchmarkBuildOn(b, 10000) } +func BenchmarkBuildOn100000(b *testing.B) { benchmarkBuildOn(b, 100000) } + +func benchmarkBuildOn(b *testing.B, size int) { + b.StopTimer() + b.ResetTimer() + + // re-use the builder suite + suite := new(BuilderSuite) + + // Copied from SetupTest. We can't use that function because suite.Assert + // is incompatible with benchmarks. + // ref: https://github.com/stretchr/testify/issues/811 + { + var err error + + suite.genesis = model.Genesis() + suite.chainID = suite.genesis.Header.ChainID + + suite.pool = herocache.NewTransactions(1000, unittest.Logger(), metrics.NewNoopCollector()) + + suite.dbdir = unittest.TempDir(b) + suite.db = unittest.BadgerDB(b, suite.dbdir) + defer func() { + err = suite.db.Close() + assert.NoError(b, err) + err = os.RemoveAll(suite.dbdir) + assert.NoError(b, err) + }() + + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + all := sutil.StorageLayer(suite.T(), suite.db) + suite.headers = all.Headers + suite.blocks = all.Blocks + suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) + + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) + stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc, suite.epochCounter) + + state, err := clusterkv.Bootstrap(suite.db, stateRoot) + assert.NoError(b, err) + + suite.state, err = clusterkv.NewMutableState(state, tracer, suite.headers, suite.payloads) + assert.NoError(b, err) + + // add some transactions to transaction pool + for i := 0; i < 3; i++ { + tx := unittest.TransactionBodyFixture() + added := suite.pool.Add(&tx) + assert.True(b, added) + } + + // create the builder + suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) + } + + // create a block history to test performance against + final := suite.genesis + for i := 0; i < size; i++ { + block := unittest.ClusterBlockWithParent(final) + err := suite.db.Update(procedure.InsertClusterBlock(&block)) + require.NoError(b, err) + + // finalize the block 80% of the time, resulting in a fork-rate of 20% + if rand.Intn(100) < 80 { + err = suite.db.Update(procedure.FinalizeClusterBlock(block.ID())) + require.NoError(b, err) + final = &block + } + } + + b.StartTimer() + for n := 0; n < b.N; n++ { + _, err := suite.builder.BuildOn(final.ID(), noopSetter) + assert.NoError(b, err) + } +} diff --git a/module/builder/consensus/builder_pebble.go b/module/builder/consensus/builder_pebble.go new file mode 100644 index 00000000000..b9a279a0dcc --- /dev/null +++ b/module/builder/consensus/builder_pebble.go @@ -0,0 +1,670 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package consensus + +import ( + "context" + "fmt" + "time" + + "github.com/dgraph-io/badger/v2" + otelTrace "go.opentelemetry.io/otel/trace" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter/id" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/fork" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/blocktimer" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// Builder is the builder for consensus block payloads. Upon providing a payload +// hash, it also memorizes which entities were included into the payload. +type Builder struct { + metrics module.MempoolMetrics + tracer module.Tracer + db *badger.DB + state protocol.ParticipantState + seals storage.Seals + headers storage.Headers + index storage.Index + blocks storage.Blocks + resultsDB storage.ExecutionResults + receiptsDB storage.ExecutionReceipts + guarPool mempool.Guarantees + sealPool mempool.IncorporatedResultSeals + recPool mempool.ExecutionTree + cfg Config +} + +// NewBuilder creates a new block builder. +func NewBuilder( + metrics module.MempoolMetrics, + db *badger.DB, + state protocol.ParticipantState, + headers storage.Headers, + seals storage.Seals, + index storage.Index, + blocks storage.Blocks, + resultsDB storage.ExecutionResults, + receiptsDB storage.ExecutionReceipts, + guarPool mempool.Guarantees, + sealPool mempool.IncorporatedResultSeals, + recPool mempool.ExecutionTree, + tracer module.Tracer, + options ...func(*Config), +) (*Builder, error) { + + blockTimer, err := blocktimer.NewBlockTimer(500*time.Millisecond, 10*time.Second) + if err != nil { + return nil, fmt.Errorf("could not create default block timer: %w", err) + } + + // initialize default config + cfg := Config{ + blockTimer: blockTimer, + maxSealCount: 100, + maxGuaranteeCount: 100, + maxReceiptCount: 200, + expiry: flow.DefaultTransactionExpiry, + } + + // apply option parameters + for _, option := range options { + option(&cfg) + } + + b := &Builder{ + metrics: metrics, + db: db, + tracer: tracer, + state: state, + headers: headers, + seals: seals, + index: index, + blocks: blocks, + resultsDB: resultsDB, + receiptsDB: receiptsDB, + guarPool: guarPool, + sealPool: sealPool, + recPool: recPool, + cfg: cfg, + } + + err = b.repopulateExecutionTree() + if err != nil { + return nil, fmt.Errorf("could not repopulate execution tree: %w", err) + } + + return b, nil +} + +// BuildOn creates a new block header on top of the provided parent, using the +// given view and applying the custom setter function to allow the caller to +// make changes to the header before storing it. +func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { + + // since we don't know the blockID when building the block we track the + // time indirectly and insert the span directly at the end + + startTime := time.Now() + + // get the collection guarantees to insert in the payload + insertableGuarantees, err := b.getInsertableGuarantees(parentID) + if err != nil { + return nil, fmt.Errorf("could not insert guarantees: %w", err) + } + + // get the receipts to insert in the payload + insertableReceipts, err := b.getInsertableReceipts(parentID) + if err != nil { + return nil, fmt.Errorf("could not insert receipts: %w", err) + } + + // get the seals to insert in the payload + insertableSeals, err := b.getInsertableSeals(parentID) + if err != nil { + return nil, fmt.Errorf("could not insert seals: %w", err) + } + + // assemble the block proposal + proposal, err := b.createProposal(parentID, + insertableGuarantees, + insertableSeals, + insertableReceipts, + setter) + if err != nil { + return nil, fmt.Errorf("could not assemble proposal: %w", err) + } + + span, ctx := b.tracer.StartBlockSpan(context.Background(), proposal.ID(), trace.CONBuilderBuildOn, otelTrace.WithTimestamp(startTime)) + defer span.End() + + err = b.state.Extend(ctx, proposal) + if err != nil { + return nil, fmt.Errorf("could not extend state with built proposal: %w", err) + } + + return proposal.Header, nil +} + +// repopulateExecutionTree restores latest state of execution tree mempool based on local chain state information. +// Repopulating of execution tree is split into two parts: +// 1) traverse backwards all finalized blocks starting from last finalized block till we reach last sealed block. [lastSealedHeight, lastFinalizedHeight] +// 2) traverse forward all unfinalized(pending) blocks starting from last finalized block. +// For each block that is being traversed we will collect execution results and add them to execution tree. +func (b *Builder) repopulateExecutionTree() error { + finalizedSnapshot := b.state.Final() + finalized, err := finalizedSnapshot.Head() + if err != nil { + return fmt.Errorf("could not retrieve finalized block: %w", err) + } + finalizedID := finalized.ID() + + // Get the latest sealed block on this fork, i.e. the highest + // block for which there is a finalized seal. + latestSeal, err := b.seals.HighestInFork(finalizedID) + if err != nil { + return fmt.Errorf("could not retrieve latest seal in fork with head %x: %w", finalizedID, err) + } + latestSealedBlockID := latestSeal.BlockID + latestSealedBlock, err := b.headers.ByBlockID(latestSealedBlockID) + if err != nil { + return fmt.Errorf("could not retrieve latest sealed block (%x): %w", latestSeal.BlockID, err) + } + sealedResult, err := b.resultsDB.ByID(latestSeal.ResultID) + if err != nil { + return fmt.Errorf("could not retrieve sealed result (%x): %w", latestSeal.ResultID, err) + } + + // prune execution tree to minimum height (while the tree is still empty, for max efficiency) + err = b.recPool.PruneUpToHeight(latestSealedBlock.Height) + if err != nil { + return fmt.Errorf("could not prune execution tree to height %d: %w", latestSealedBlock.Height, err) + } + + // At initialization, the execution tree is empty. However, during normal operations, we + // generally query the tree for "all receipts, whose results are derived from the latest + // sealed and finalized result". This requires the execution tree to know what the latest + // sealed and finalized result is, so we add it here. + // Note: we only add the sealed and finalized result, without any Execution Receipts. This + // is sufficient to create a vertex in the tree. Thereby, we can traverse the tree, starting + // from the sealed and finalized result, to find derived results and their respective receipts. + err = b.recPool.AddResult(sealedResult, latestSealedBlock) + if err != nil { + return fmt.Errorf("failed to add sealed result as vertex to ExecutionTree (%x): %w", latestSeal.ResultID, err) + } + + // receiptCollector adds _all known_ receipts for the given block to the execution tree + receiptCollector := func(header *flow.Header) error { + receipts, err := b.receiptsDB.ByBlockID(header.ID()) + if err != nil { + return fmt.Errorf("could not retrieve execution reciepts for block %x: %w", header.ID(), err) + } + for _, receipt := range receipts { + _, err = b.recPool.AddReceipt(receipt, header) + if err != nil { + return fmt.Errorf("could not add receipt (%x) to execution tree: %w", receipt.ID(), err) + } + } + return nil + } + + // Traverse chain backwards and add all known receipts for any finalized, unsealed block to the execution tree. + // Thereby, we add superset of all unsealed execution results to the execution tree. + err = fork.TraverseBackward(b.headers, finalizedID, receiptCollector, fork.ExcludingBlock(latestSealedBlockID)) + if err != nil { + return fmt.Errorf("failed to traverse unsealed, finalized blocks: %w", err) + } + + // At this point execution tree is filled with all results for blocks (lastSealedBlock, lastFinalizedBlock]. + // Now, we add all known receipts for any valid block that descends from the latest finalized block: + validPending, err := finalizedSnapshot.Descendants() + if err != nil { + return fmt.Errorf("could not retrieve valid pending blocks from finalized snapshot: %w", err) + } + for _, blockID := range validPending { + block, err := b.headers.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve header for unfinalized block %x: %w", blockID, err) + } + err = receiptCollector(block) + if err != nil { + return fmt.Errorf("failed to add receipts for unfinalized block %x at height %d: %w", blockID, block.Height, err) + } + } + + return nil +} + +// getInsertableGuarantees returns the list of CollectionGuarantees that should +// be inserted in the next payload. It looks in the collection mempool and +// applies the following filters: +// +// 1) If it was already included in the fork, skip. +// +// 2) If it references an unknown block, skip. +// +// 3) If the referenced block has an expired height, skip. +// +// 4) Otherwise, this guarantee can be included in the payload. +func (b *Builder) getInsertableGuarantees(parentID flow.Identifier) ([]*flow.CollectionGuarantee, error) { + + // we look back only as far as the expiry limit for the current height we + // are building for; any guarantee with a reference block before that can + // not be included anymore anyway + parent, err := b.headers.ByBlockID(parentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve parent: %w", err) + } + height := parent.Height + 1 + limit := height - uint64(b.cfg.expiry) + if limit > height { // overflow check + limit = 0 + } + + // look up the root height so we don't look too far back + // initially this is the genesis block height (aka 0). + var rootHeight uint64 + err = b.db.View(operation.RetrieveRootHeight(&rootHeight)) + if err != nil { + return nil, fmt.Errorf("could not retrieve root block height: %w", err) + } + if limit < rootHeight { + limit = rootHeight + } + + // blockLookup keeps track of the blocks from limit to parent + blockLookup := make(map[flow.Identifier]struct{}) + + // receiptLookup keeps track of the receipts contained in blocks between + // limit and parent + receiptLookup := make(map[flow.Identifier]struct{}) + + // loop through the fork backwards, from parent to limit (inclusive), + // and keep track of blocks and collections visited on the way + forkScanner := func(header *flow.Header) error { + ancestorID := header.ID() + blockLookup[ancestorID] = struct{}{} + + index, err := b.index.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not get ancestor payload (%x): %w", ancestorID, err) + } + + for _, collID := range index.CollectionIDs { + receiptLookup[collID] = struct{}{} + } + + return nil + } + err = fork.TraverseBackward(b.headers, parentID, forkScanner, fork.IncludingHeight(limit)) + if err != nil { + return nil, fmt.Errorf("internal error building set of CollectionGuarantees on fork: %w", err) + } + + // go through mempool and collect valid collections + var guarantees []*flow.CollectionGuarantee + for _, guarantee := range b.guarPool.All() { + // add at most number of collection guarantees in a new block proposal + // in order to prevent the block payload from being too big or computationally heavy for the + // execution nodes + if uint(len(guarantees)) >= b.cfg.maxGuaranteeCount { + break + } + + collID := guarantee.ID() + + // skip collections that are already included in a block on the fork + _, duplicated := receiptLookup[collID] + if duplicated { + continue + } + + // skip collections for blocks that are not within the limit + _, ok := blockLookup[guarantee.ReferenceBlockID] + if !ok { + continue + } + + guarantees = append(guarantees, guarantee) + } + + return guarantees, nil +} + +// getInsertableSeals returns the list of Seals from the mempool that should be +// inserted in the next payload. +// Per protocol definition, a specific result is only incorporated _once_ in each fork. +// Specifically, the result is incorporated in the block that contains a receipt committing +// to a result for the _first time_ in the respective fork. +// We can seal a result if and only if _all_ of the following conditions are satisfied: +// +// - (0) We have collected a sufficient number of approvals for each of the result's chunks. +// - (1) The result must have been previously incorporated in the fork, which we are extending. +// Note: The protocol dictates that all incorporated results must be for ancestor blocks +// in the respective fork. Hence, a result being incorporated in the fork, implies +// that the result must be for a block in this fork. +// - (2) The result must be for an _unsealed_ block. +// - (3) The result's parent must have been previously sealed (either by a seal in an ancestor +// block or by a seal included earlier in the block that we are constructing). +// +// To limit block size, we cap the number of seals to maxSealCount. +func (b *Builder) getInsertableSeals(parentID flow.Identifier) ([]*flow.Seal, error) { + // get the latest seal in the fork, which we are extending and + // the corresponding block, whose result is sealed + // Note: the last seal might not be included in a finalized block yet + lastSeal, err := b.seals.HighestInFork(parentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve latest seal in the fork, which we are extending: %w", err) + } + latestSealedBlockID := lastSeal.BlockID + latestSealedBlock, err := b.headers.ByBlockID(latestSealedBlockID) + if err != nil { + return nil, fmt.Errorf("could not retrieve sealed block %x: %w", lastSeal.BlockID, err) + } + latestSealedHeight := latestSealedBlock.Height + + // STEP I: Collect the seals for all results that satisfy (0), (1), and (2). + // The will give us a _superset_ of all seals that can be included. + // Implementation: + // * We walk the fork backwards and check each block for incorporated results. + // - Therefore, all results that we encounter satisfy condition (1). + // * We only consider results, whose executed block has a height _strictly larger_ + // than the lastSealedHeight. + // - Thereby, we guarantee that condition (2) is satisfied. + // * We only consider results for which we have a candidate seals in the sealPool. + // - Thereby, we guarantee that condition (0) is satisfied, because candidate seals + // are only generated and stored in the mempool once sufficient approvals are collected. + // Furthermore, condition (2) imposes a limit on how far we have to walk back: + // * A result can only be incorporated in a child of the block that it computes. + // Therefore, we only have to inspect the results incorporated in unsealed blocks. + sealsSuperset := make(map[uint64][]*flow.IncorporatedResultSeal) // map: executedBlock.Height -> candidate Seals + sealCollector := func(header *flow.Header) error { + blockID := header.ID() + if blockID == parentID { + // Important protocol edge case: There must be at least one block in between the block incorporating + // a result and the block sealing the result. This is because we need the Source of Randomness for + // the block that _incorporates_ the result, to compute the verifier assignment. Therefore, we require + // that the block _incorporating_ the result has at least one child in the fork, _before_ we include + // the seal. Thereby, we guarantee that a verifier assignment can be computed without needing + // information from the block that we are just constructing. Hence, we don't consider results for + // sealing that were incorporated in the immediate parent which we are extending. + return nil + } + + index, err := b.index.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve index for block %x: %w", blockID, err) + } + + // enforce condition (1): only consider seals for results that are incorporated in the fork + for _, resultID := range index.ResultIDs { + result, err := b.resultsDB.ByID(resultID) + if err != nil { + return fmt.Errorf("could not retrieve execution result %x: %w", resultID, err) + } + + // re-assemble the IncorporatedResult because we need its ID to + // check if it is in the seal mempool. + incorporatedResult := flow.NewIncorporatedResult( + blockID, + result, + ) + + // enforce condition (0): candidate seals are only constructed once sufficient + // approvals have been collected. Hence, any incorporated result for which we + // find a candidate seal satisfies condition (0) + irSeal, ok := b.sealPool.ByID(incorporatedResult.ID()) + if !ok { + continue + } + + // enforce condition (2): the block is unsealed (in this fork) if and only if + // its height is _strictly larger_ than the lastSealedHeight. + executedBlock, err := b.headers.ByBlockID(incorporatedResult.Result.BlockID) + if err != nil { + return fmt.Errorf("could not get header of block %x: %w", incorporatedResult.Result.BlockID, err) + } + if executedBlock.Height <= latestSealedHeight { + continue + } + + // The following is a subtle but important protocol edge case: There can be multiple + // candidate seals for the same block. We have to include all to guarantee sealing liveness! + sealsSuperset[executedBlock.Height] = append(sealsSuperset[executedBlock.Height], irSeal) + } + + return nil + } + err = fork.TraverseBackward(b.headers, parentID, sealCollector, fork.ExcludingBlock(latestSealedBlockID)) + if err != nil { + return nil, fmt.Errorf("internal error traversing unsealed section of fork: %w", err) + } + // All the seals in sealsSuperset are for results that satisfy (0), (1), and (2). + + // STEP II: Select only the seals from sealsSuperset that also satisfy condition (3). + // We do this by starting with the last sealed result in the fork. Then, we check whether we + // have a seal for the child block (at latestSealedBlock.Height +1), which connects to the + // sealed result. If we find such a seal, we can now consider the child block sealed. + // We continue until we stop finding a seal for the child. + seals := make([]*flow.Seal, 0, len(sealsSuperset)) + for { + // cap the number of seals + if uint(len(seals)) >= b.cfg.maxSealCount { + break + } + + // enforce condition (3): + candidateSeal, ok := connectingSeal(sealsSuperset[latestSealedHeight+1], lastSeal) + if !ok { + break + } + seals = append(seals, candidateSeal) + lastSeal = candidateSeal + latestSealedHeight += 1 + } + return seals, nil +} + +// connectingSeal looks through `sealsForNextBlock`. It checks whether the +// sealed result directly descends from the lastSealed result. +func connectingSeal(sealsForNextBlock []*flow.IncorporatedResultSeal, lastSealed *flow.Seal) (*flow.Seal, bool) { + for _, candidateSeal := range sealsForNextBlock { + if candidateSeal.IncorporatedResult.Result.PreviousResultID == lastSealed.ResultID { + return candidateSeal.Seal, true + } + } + return nil, false +} + +type InsertableReceipts struct { + receipts []*flow.ExecutionReceiptMeta + results []*flow.ExecutionResult +} + +// getInsertableReceipts constructs: +// - (i) the meta information of the ExecutionReceipts (i.e. ExecutionReceiptMeta) +// that should be inserted in the next payload +// - (ii) the ExecutionResults the receipts from step (i) commit to +// (deduplicated w.r.t. the block under construction as well as ancestor blocks) +// +// It looks in the receipts mempool and applies the following filter: +// +// 1) If it doesn't correspond to an unsealed block on the fork, skip it. +// +// 2) If it was already included in the fork, skip it. +// +// 3) Otherwise, this receipt can be included in the payload. +// +// Receipts have to be ordered by block height. +func (b *Builder) getInsertableReceipts(parentID flow.Identifier) (*InsertableReceipts, error) { + + // Get the latest sealed block on this fork, ie the highest block for which + // there is a seal in this fork. This block is not necessarily finalized. + latestSeal, err := b.seals.HighestInFork(parentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve parent seal (%x): %w", parentID, err) + } + sealedBlockID := latestSeal.BlockID + + // ancestors is used to keep the IDs of the ancestor blocks we iterate through. + // We use it to skip receipts that are not for unsealed blocks in the fork. + ancestors := make(map[flow.Identifier]struct{}) + + // includedReceipts is a set of all receipts that are contained in unsealed blocks along the fork. + includedReceipts := make(map[flow.Identifier]struct{}) + + // includedResults is a set of all unsealed results that were incorporated into fork + includedResults := make(map[flow.Identifier]struct{}) + + // loop through the fork backwards, from parent to last sealed (including), + // and keep track of blocks and receipts visited on the way. + forkScanner := func(ancestor *flow.Header) error { + ancestorID := ancestor.ID() + ancestors[ancestorID] = struct{}{} + + index, err := b.index.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not get payload index of block %x: %w", ancestorID, err) + } + for _, recID := range index.ReceiptIDs { + includedReceipts[recID] = struct{}{} + } + for _, resID := range index.ResultIDs { + includedResults[resID] = struct{}{} + } + + return nil + } + err = fork.TraverseBackward(b.headers, parentID, forkScanner, fork.IncludingBlock(sealedBlockID)) + if err != nil { + return nil, fmt.Errorf("internal error building set of CollectionGuarantees on fork: %w", err) + } + + isResultForUnsealedBlock := isResultForBlock(ancestors) + isReceiptUniqueAndUnsealed := isNoDupAndNotSealed(includedReceipts, sealedBlockID) + // find all receipts: + // 1) whose result connects all the way to the last sealed result + // 2) is unique (never seen in unsealed blocks) + receipts, err := b.recPool.ReachableReceipts(latestSeal.ResultID, isResultForUnsealedBlock, isReceiptUniqueAndUnsealed) + // Occurrence of UnknownExecutionResultError: + // Populating the execution with receipts from incoming blocks happens concurrently in + // matching.Core. Hence, the following edge case can occur (rarely): matching.Core is + // just in the process of populating the Execution Tree with the receipts from the + // latest blocks, while the builder is already trying to build on top. In this rare + // situation, the Execution Tree might not yet know the latest sealed result. + // TODO: we should probably remove this edge case by _synchronously_ populating + // the Execution Tree in the Fork's finalizationCallback + if err != nil && !mempool.IsUnknownExecutionResultError(err) { + return nil, fmt.Errorf("failed to retrieve reachable receipts from memool: %w", err) + } + + insertables := toInsertables(receipts, includedResults, b.cfg.maxReceiptCount) + return insertables, nil +} + +// toInsertables separates the provided receipts into ExecutionReceiptMeta and +// ExecutionResult. Results that are in includedResults are skipped. +// We also limit the number of receipts to maxReceiptCount. +func toInsertables(receipts []*flow.ExecutionReceipt, includedResults map[flow.Identifier]struct{}, maxReceiptCount uint) *InsertableReceipts { + results := make([]*flow.ExecutionResult, 0) + + count := uint(len(receipts)) + // don't collect more than maxReceiptCount receipts + if count > maxReceiptCount { + count = maxReceiptCount + } + + filteredReceipts := make([]*flow.ExecutionReceiptMeta, 0, count) + + for i := uint(0); i < count; i++ { + receipt := receipts[i] + meta := receipt.Meta() + resultID := meta.ResultID + if _, inserted := includedResults[resultID]; !inserted { + results = append(results, &receipt.ExecutionResult) + includedResults[resultID] = struct{}{} + } + + filteredReceipts = append(filteredReceipts, meta) + } + + return &InsertableReceipts{ + receipts: filteredReceipts, + results: results, + } +} + +// createProposal assembles a block with the provided header and payload +// information +func (b *Builder) createProposal(parentID flow.Identifier, + guarantees []*flow.CollectionGuarantee, + seals []*flow.Seal, + insertableReceipts *InsertableReceipts, + setter func(*flow.Header) error) (*flow.Block, error) { + + // build the payload so we can get the hash + payload := &flow.Payload{ + Guarantees: guarantees, + Seals: seals, + Receipts: insertableReceipts.receipts, + Results: insertableReceipts.results, + } + + parent, err := b.headers.ByBlockID(parentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve parent: %w", err) + } + + timestamp := b.cfg.blockTimer.Build(parent.Timestamp) + + // construct default block on top of the provided parent + header := &flow.Header{ + ChainID: parent.ChainID, + ParentID: parentID, + Height: parent.Height + 1, + Timestamp: timestamp, + PayloadHash: payload.Hash(), + } + + // apply the custom fields setter of the consensus algorithm + err = setter(header) + if err != nil { + return nil, fmt.Errorf("could not apply setter: %w", err) + } + + proposal := &flow.Block{ + Header: header, + Payload: payload, + } + + return proposal, nil +} + +// isResultForBlock constructs a mempool.BlockFilter that accepts only blocks whose ID is part of the given set. +func isResultForBlock(blockIDs map[flow.Identifier]struct{}) mempool.BlockFilter { + blockIdFilter := id.InSet(blockIDs) + return func(h *flow.Header) bool { + return blockIdFilter(h.ID()) + } +} + +// isNoDupAndNotSealed constructs a mempool.ReceiptFilter for discarding receipts that +// * are duplicates +// * or are for the sealed block +func isNoDupAndNotSealed(includedReceipts map[flow.Identifier]struct{}, sealedBlockID flow.Identifier) mempool.ReceiptFilter { + return func(receipt *flow.ExecutionReceipt) bool { + if _, duplicate := includedReceipts[receipt.ID()]; duplicate { + return false + } + if receipt.ExecutionResult.BlockID == sealedBlockID { + return false + } + return true + } +} diff --git a/module/builder/consensus/builder_pebble_test.go b/module/builder/consensus/builder_pebble_test.go new file mode 100644 index 00000000000..d8f82c8eda8 --- /dev/null +++ b/module/builder/consensus/builder_pebble_test.go @@ -0,0 +1,1463 @@ +package consensus + +import ( + "math/rand" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/model/flow" + mempoolAPIs "github.com/onflow/flow-go/module/mempool" + mempoolImpl "github.com/onflow/flow-go/module/mempool/consensus" + mempool "github.com/onflow/flow-go/module/mempool/mock" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/trace" + realproto "github.com/onflow/flow-go/state/protocol" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storerr "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + storage "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestConsensusBuilder(t *testing.T) { + suite.Run(t, new(BuilderSuite)) +} + +type BuilderSuite struct { + suite.Suite + + // test helpers + firstID flow.Identifier // first block in the range we look at + finalID flow.Identifier // last finalized block + parentID flow.Identifier // Parent block we build on + finalizedBlockIDs []flow.Identifier // blocks between first and final + pendingBlockIDs []flow.Identifier // blocks between final and parent + resultForBlock map[flow.Identifier]*flow.ExecutionResult // map: BlockID -> Execution Result + resultByID map[flow.Identifier]*flow.ExecutionResult // map: result ID -> Execution Result + receiptsByID map[flow.Identifier]*flow.ExecutionReceipt // map: receipt ID -> ExecutionReceipt + receiptsByBlockID map[flow.Identifier]flow.ExecutionReceiptList // map: block ID -> flow.ExecutionReceiptList + + // used to populate and test the seal mempool + chain []*flow.Seal // chain of seals starting first + irsList []*flow.IncorporatedResultSeal // chain of IncorporatedResultSeals + irsMap map[flow.Identifier]*flow.IncorporatedResultSeal // index for irsList + + // mempools consumed by builder + pendingGuarantees []*flow.CollectionGuarantee + pendingReceipts []*flow.ExecutionReceipt + pendingSeals map[flow.Identifier]*flow.IncorporatedResultSeal // storage for the seal mempool + + // storage for dbs + headers map[flow.Identifier]*flow.Header + index map[flow.Identifier]*flow.Index + blocks map[flow.Identifier]*flow.Block + blockChildren map[flow.Identifier][]flow.Identifier // ids of children blocks + + lastSeal *flow.Seal + + // real dependencies + dir string + db *badger.DB + sentinel uint64 + setter func(*flow.Header) error + + // mocked dependencies + state *protocol.ParticipantState + headerDB *storage.Headers + sealDB *storage.Seals + indexDB *storage.Index + blockDB *storage.Blocks + resultDB *storage.ExecutionResults + receiptsDB *storage.ExecutionReceipts + + guarPool *mempool.Guarantees + sealPool *mempool.IncorporatedResultSeals + recPool *mempool.ExecutionTree + + // tracking behaviour + assembled *flow.Payload // built payload + + // component under test + build *Builder +} + +func (bs *BuilderSuite) storeBlock(block *flow.Block) { + bs.headers[block.ID()] = block.Header + bs.blocks[block.ID()] = block + bs.index[block.ID()] = block.Payload.Index() + bs.blockChildren[block.Header.ParentID] = append(bs.blockChildren[block.Header.ParentID], block.ID()) + for _, result := range block.Payload.Results { + bs.resultByID[result.ID()] = result + } +} + +// createAndRecordBlock creates a new block chained to the previous block. +// The new block contains a receipt for a result of the previous +// block, which is also used to create a seal for the previous block. The seal +// and the result are combined in an IncorporatedResultSeal which is a candidate +// for the seals mempool. +func (bs *BuilderSuite) createAndRecordBlock(parentBlock *flow.Block, candidateSealForParent bool) *flow.Block { + block := unittest.BlockWithParentFixture(parentBlock.Header) + + // Create a receipt for a result of the parentBlock block, + // and add it to the payload. The corresponding IncorporatedResult will be used to + // seal the parentBlock, and to create an IncorporatedResultSeal for the seal mempool. + var incorporatedResultForPrevBlock *flow.IncorporatedResult + previousResult, found := bs.resultForBlock[parentBlock.ID()] + if !found { + panic("missing execution result for parent") + } + receipt := unittest.ExecutionReceiptFixture(unittest.WithResult(previousResult)) + block.Payload.Receipts = append(block.Payload.Receipts, receipt.Meta()) + block.Payload.Results = append(block.Payload.Results, &receipt.ExecutionResult) + + incorporatedResultForPrevBlock = unittest.IncorporatedResult.Fixture( + unittest.IncorporatedResult.WithResult(previousResult), + unittest.IncorporatedResult.WithIncorporatedBlockID(block.ID()), + ) + + result := unittest.ExecutionResultFixture( + unittest.WithBlock(block), + unittest.WithPreviousResult(*previousResult), + ) + + bs.resultForBlock[result.BlockID] = result + bs.resultByID[result.ID()] = result + bs.receiptsByID[receipt.ID()] = receipt + bs.receiptsByBlockID[receipt.ExecutionResult.BlockID] = append(bs.receiptsByBlockID[receipt.ExecutionResult.BlockID], receipt) + + // record block in dbs + bs.storeBlock(block) + + if candidateSealForParent { + // seal the parentBlock block with the result included in this block. + bs.chainSeal(incorporatedResultForPrevBlock) + } + + return block +} + +// Create a seal for the result's block. The corresponding +// IncorporatedResultSeal, which ties the seal to the incorporated result it +// seals, is also recorded for future access. +func (bs *BuilderSuite) chainSeal(incorporatedResult *flow.IncorporatedResult) { + incorporatedResultSeal := unittest.IncorporatedResultSeal.Fixture( + unittest.IncorporatedResultSeal.WithResult(incorporatedResult.Result), + unittest.IncorporatedResultSeal.WithIncorporatedBlockID(incorporatedResult.IncorporatedBlockID), + ) + + bs.chain = append(bs.chain, incorporatedResultSeal.Seal) + bs.irsMap[incorporatedResultSeal.ID()] = incorporatedResultSeal + bs.irsList = append(bs.irsList, incorporatedResultSeal) +} + +// SetupTest constructs the following chain of blocks: +// +// [first] <- [F0] <- [F1] <- [F2] <- [F3] <- [final] <- [A0] <- [A1] <- [A2] <- [A3] <- [parent] +// +// Where block +// - [first] is sealed and finalized +// - [F0] ... [F4] and [final] are finalized, unsealed blocks with candidate seals are included in mempool +// - [A0] ... [A2] are non-finalized, unsealed blocks with candidate seals are included in mempool +// - [A3] and [parent] are non-finalized, unsealed blocks _without_ candidate seals +// +// Each block incorporates the result for its immediate parent. +// +// Note: In the happy path, the blocks [A3] and [parent] will not have candidate seal for the following reason: +// For the verifiers to start checking a result R, they need a source of randomness for the block _incorporating_ +// result R. The result for block [A3] is incorporated in [parent], which does _not_ have a child yet. +func (bs *BuilderSuite) SetupTest() { + + // set up no-op dependencies + noopMetrics := metrics.NewNoopCollector() + noopTracer := trace.NewNoopTracer() + + // set up test parameters + numFinalizedBlocks := 4 + numPendingBlocks := 4 + + // reset test helpers + bs.pendingBlockIDs = nil + bs.finalizedBlockIDs = nil + bs.resultForBlock = make(map[flow.Identifier]*flow.ExecutionResult) + bs.resultByID = make(map[flow.Identifier]*flow.ExecutionResult) + bs.receiptsByID = make(map[flow.Identifier]*flow.ExecutionReceipt) + bs.receiptsByBlockID = make(map[flow.Identifier]flow.ExecutionReceiptList) + + bs.chain = nil + bs.irsMap = make(map[flow.Identifier]*flow.IncorporatedResultSeal) + bs.irsList = nil + + // initialize the pools + bs.pendingGuarantees = nil + bs.pendingSeals = nil + bs.pendingReceipts = nil + + // initialise the dbs + bs.lastSeal = nil + bs.headers = make(map[flow.Identifier]*flow.Header) + //bs.heights = make(map[uint64]*flow.Header) + bs.index = make(map[flow.Identifier]*flow.Index) + bs.blocks = make(map[flow.Identifier]*flow.Block) + bs.blockChildren = make(map[flow.Identifier][]flow.Identifier) + + // initialize behaviour tracking + bs.assembled = nil + + // Construct the [first] block: + first := unittest.BlockFixture() + bs.storeBlock(&first) + bs.firstID = first.ID() + firstResult := unittest.ExecutionResultFixture(unittest.WithBlock(&first)) + bs.lastSeal = unittest.Seal.Fixture(unittest.Seal.WithResult(firstResult)) + bs.resultForBlock[firstResult.BlockID] = firstResult + bs.resultByID[firstResult.ID()] = firstResult + + // Construct finalized blocks [F0] ... [F4] + previous := &first + for n := 0; n < numFinalizedBlocks; n++ { + finalized := bs.createAndRecordBlock(previous, n > 0) // Do not construct candidate seal for [first], as it is already sealed + bs.finalizedBlockIDs = append(bs.finalizedBlockIDs, finalized.ID()) + previous = finalized + } + + // Construct the last finalized block [final] + final := bs.createAndRecordBlock(previous, true) + bs.finalID = final.ID() + + // Construct the pending (i.e. unfinalized) ancestors [A0], ..., [A3] + previous = final + for n := 0; n < numPendingBlocks; n++ { + pending := bs.createAndRecordBlock(previous, true) + bs.pendingBlockIDs = append(bs.pendingBlockIDs, pending.ID()) + previous = pending + } + + // Construct [parent] block; but do _not_ add candidate seal for its parent + parent := bs.createAndRecordBlock(previous, false) + bs.parentID = parent.ID() + + // set up temporary database for tests + bs.db, bs.dir = unittest.TempBadgerDB(bs.T()) + + err := bs.db.Update(operation.InsertFinalizedHeight(final.Header.Height)) + bs.Require().NoError(err) + err = bs.db.Update(operation.IndexBlockHeight(final.Header.Height, bs.finalID)) + bs.Require().NoError(err) + + err = bs.db.Update(operation.InsertRootHeight(13)) + bs.Require().NoError(err) + + err = bs.db.Update(operation.InsertSealedHeight(first.Header.Height)) + bs.Require().NoError(err) + err = bs.db.Update(operation.IndexBlockHeight(first.Header.Height, first.ID())) + bs.Require().NoError(err) + + bs.sentinel = 1337 + + bs.setter = func(header *flow.Header) error { + header.View = 1337 + return nil + } + + bs.state = &protocol.ParticipantState{} + bs.state.On("Extend", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + block := args.Get(1).(*flow.Block) + bs.Assert().Equal(bs.sentinel, block.Header.View) + bs.assembled = block.Payload + }).Return(nil) + bs.state.On("Final").Return(func() realproto.Snapshot { + if block, ok := bs.blocks[bs.finalID]; ok { + snapshot := unittest.StateSnapshotForKnownBlock(block.Header, nil) + snapshot.On("Descendants").Return(bs.blockChildren[bs.finalID], nil) + return snapshot + } + return unittest.StateSnapshotForUnknownBlock() + }) + + // set up storage mocks for tests + bs.sealDB = &storage.Seals{} + bs.sealDB.On("HighestInFork", mock.Anything).Return(bs.lastSeal, nil) + + bs.headerDB = &storage.Headers{} + bs.headerDB.On("ByBlockID", mock.Anything).Return( + func(blockID flow.Identifier) *flow.Header { + return bs.headers[blockID] + }, + func(blockID flow.Identifier) error { + _, exists := bs.headers[blockID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + + bs.indexDB = &storage.Index{} + bs.indexDB.On("ByBlockID", mock.Anything).Return( + func(blockID flow.Identifier) *flow.Index { + return bs.index[blockID] + }, + func(blockID flow.Identifier) error { + _, exists := bs.index[blockID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + + bs.blockDB = &storage.Blocks{} + bs.blockDB.On("ByID", mock.Anything).Return( + func(blockID flow.Identifier) *flow.Block { + return bs.blocks[blockID] + }, + func(blockID flow.Identifier) error { + _, exists := bs.blocks[blockID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + + bs.resultDB = &storage.ExecutionResults{} + bs.resultDB.On("ByID", mock.Anything).Return( + func(resultID flow.Identifier) *flow.ExecutionResult { + return bs.resultByID[resultID] + }, + func(resultID flow.Identifier) error { + _, exists := bs.resultByID[resultID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + + bs.receiptsDB = &storage.ExecutionReceipts{} + bs.receiptsDB.On("ByID", mock.Anything).Return( + func(receiptID flow.Identifier) *flow.ExecutionReceipt { + return bs.receiptsByID[receiptID] + }, + func(receiptID flow.Identifier) error { + _, exists := bs.receiptsByID[receiptID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + bs.receiptsDB.On("ByBlockID", mock.Anything).Return( + func(blockID flow.Identifier) flow.ExecutionReceiptList { + return bs.receiptsByBlockID[blockID] + }, + func(blockID flow.Identifier) error { + _, exists := bs.receiptsByBlockID[blockID] + if !exists { + return storerr.ErrNotFound + } + return nil + }, + ) + + // set up memory pool mocks for tests + bs.guarPool = &mempool.Guarantees{} + bs.guarPool.On("Size").Return(uint(0)) // only used by metrics + bs.guarPool.On("All").Return( + func() []*flow.CollectionGuarantee { + return bs.pendingGuarantees + }, + ) + + bs.sealPool = &mempool.IncorporatedResultSeals{} + bs.sealPool.On("Size").Return(uint(0)) // only used by metrics + bs.sealPool.On("All").Return( + func() []*flow.IncorporatedResultSeal { + res := make([]*flow.IncorporatedResultSeal, 0, len(bs.pendingSeals)) + for _, ps := range bs.pendingSeals { + res = append(res, ps) + } + return res + }, + ) + bs.sealPool.On("ByID", mock.Anything).Return( + func(id flow.Identifier) *flow.IncorporatedResultSeal { + return bs.pendingSeals[id] + }, + func(id flow.Identifier) bool { + _, exists := bs.pendingSeals[id] + return exists + }, + ) + + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("PruneUpToHeight", mock.Anything).Return(nil).Maybe() + bs.recPool.On("Size").Return(uint(0)).Maybe() // used for metrics only + bs.recPool.On("AddResult", mock.Anything, mock.Anything).Return(nil).Maybe() + bs.recPool.On("AddReceipt", mock.Anything, mock.Anything).Return(false, nil).Maybe() + bs.recPool.On("ReachableReceipts", mock.Anything, mock.Anything, mock.Anything).Return( + func(resultID flow.Identifier, blockFilter mempoolAPIs.BlockFilter, receiptFilter mempoolAPIs.ReceiptFilter) []*flow.ExecutionReceipt { + return bs.pendingReceipts + }, + nil, + ) + + // initialize the builder + bs.build, err = NewBuilder( + noopMetrics, + bs.db, + bs.state, + bs.headerDB, + bs.sealDB, + bs.indexDB, + bs.blockDB, + bs.resultDB, + bs.receiptsDB, + bs.guarPool, + bs.sealPool, + bs.recPool, + noopTracer, + ) + require.NoError(bs.T(), err) + + bs.build.cfg.expiry = 11 +} + +func (bs *BuilderSuite) TearDownTest() { + err := bs.db.Close() + bs.Assert().NoError(err) + err = os.RemoveAll(bs.dir) + bs.Assert().NoError(err) +} + +func (bs *BuilderSuite) TestPayloadEmptyValid() { + + // we should build an empty block with default setup + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") + bs.Assert().Empty(bs.assembled.Seals, "should have no seals in payload with empty mempool") +} + +func (bs *BuilderSuite) TestPayloadGuaranteeValid() { + + // add sixteen guarantees to the pool + bs.pendingGuarantees = unittest.CollectionGuaranteesFixture(16, unittest.WithCollRef(bs.finalID)) + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(bs.pendingGuarantees, bs.assembled.Guarantees, "should have guarantees from mempool in payload") +} + +func (bs *BuilderSuite) TestPayloadGuaranteeDuplicate() { + + // create some valid guarantees + valid := unittest.CollectionGuaranteesFixture(4, unittest.WithCollRef(bs.finalID)) + + forkBlocks := append(bs.finalizedBlockIDs, bs.pendingBlockIDs...) + + // create some duplicate guarantees and add to random blocks on the fork + duplicated := unittest.CollectionGuaranteesFixture(12, unittest.WithCollRef(bs.finalID)) + for _, guarantee := range duplicated { + blockID := forkBlocks[rand.Intn(len(forkBlocks))] + index := bs.index[blockID] + index.CollectionIDs = append(index.CollectionIDs, guarantee.ID()) + bs.index[blockID] = index + } + + // add sixteen guarantees to the pool + bs.pendingGuarantees = append(valid, duplicated...) + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid guarantees from mempool in payload") +} + +func (bs *BuilderSuite) TestPayloadGuaranteeReferenceUnknown() { + + // create 12 valid guarantees + valid := unittest.CollectionGuaranteesFixture(12, unittest.WithCollRef(bs.finalID)) + + // create 4 guarantees with unknown reference + unknown := unittest.CollectionGuaranteesFixture(4, unittest.WithCollRef(unittest.IdentifierFixture())) + + // add all guarantees to the pool + bs.pendingGuarantees = append(valid, unknown...) + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid from mempool in payload") +} + +func (bs *BuilderSuite) TestPayloadGuaranteeReferenceExpired() { + + // create 12 valid guarantees + valid := unittest.CollectionGuaranteesFixture(12, unittest.WithCollRef(bs.finalID)) + + // create 4 expired guarantees + header := unittest.BlockHeaderFixture() + header.Height = bs.headers[bs.finalID].Height - 12 + bs.headers[header.ID()] = header + expired := unittest.CollectionGuaranteesFixture(4, unittest.WithCollRef(header.ID())) + + // add all guarantees to the pool + bs.pendingGuarantees = append(valid, expired...) + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid from mempool in payload") +} + +// TestPayloadSeals_AllValid checks that builder seals as many blocks as possible (happy path): +// +// [first] <- [F0] <- [F1] <- [F2] <- [F3] <- [final] <- [A0] <- [A1] <- [A2] <- [A3] <- [parent] +// +// Where block +// - [first] is sealed and finalized +// - [F0] ... [F4] and [final] are finalized, unsealed blocks with candidate seals are included in mempool +// - [A0] ... [A2] are non-finalized, unsealed blocks with candidate seals are included in mempool +// - [A3] and [parent] are non-finalized, unsealed blocks _without_ candidate seals +// +// Expected behaviour: +// - builder should include seals [F0], ..., [A4] +// - note: Block [A3] will not have a seal in the happy path for the following reason: +// In our example, the result for block A3 is incorporated in block A4. But, for the verifiers to start +// their work, they need a child block of A4, because the child contains the source of randomness for +// A4. But we are just constructing this child right now. Hence, the verifiers couldn't have checked +// the result for A3. +func (bs *BuilderSuite) TestPayloadSeals_AllValid() { + // Populate seals mempool with valid chain of seals for blocks [F0], ..., [A2] + bs.pendingSeals = bs.irsMap + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") + bs.Assert().ElementsMatch(bs.chain, bs.assembled.Seals, "should have included valid chain of seals") +} + +// TestPayloadSeals_Limit verifies that builder does not exceed maxSealLimit +func (bs *BuilderSuite) TestPayloadSeals_Limit() { + // use valid chain of seals in mempool + bs.pendingSeals = bs.irsMap + + // change maxSealCount to one less than the number of items in the mempool + limit := uint(2) + bs.build.cfg.maxSealCount = limit + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") + bs.Assert().Equal(bs.chain[:limit], bs.assembled.Seals, "should have excluded seals above maxSealCount") +} + +// TestPayloadSeals_OnlyFork checks that the builder only includes seals corresponding +// to blocks on the current fork (and _not_ seals for sealable blocks on other forks) +func (bs *BuilderSuite) TestPayloadSeals_OnlyFork() { + // in the test setup, we already created a single fork + // [first] <- [F0] <- [F1] <- [F2] <- [F3] <- [final] <- [A0] <- [A1] <- [A2] .. + // For this test, we add fork: ^ + // └--- [B0] <- [B1] <- ....<- [B6] <- [B7] + // Where block + // * [first] is sealed and finalized + // * [F0] ... [F4] and [final] are finalized, unsealed blocks with candidate seals are included in mempool + // * [A0] ... [A2] are non-finalized, unsealed blocks with candidate seals are included in mempool + forkHead := bs.blocks[bs.finalID] + for i := 0; i < 8; i++ { + // Usually, the blocks [B6] and [B7] will not have candidate seal for the following reason: + // For the verifiers to start checking a result R, they need a source of randomness for the block _incorporating_ + // result R. The result for block [B6] is incorporated in [B7], which does _not_ have a child yet. + forkHead = bs.createAndRecordBlock(forkHead, i < 6) + } + + bs.pendingSeals = bs.irsMap + _, err := bs.build.BuildOn(forkHead.ID(), bs.setter) + bs.Require().NoError(err) + + // expected seals: [F0] <- ... <- [final] <- [B0] <- ... <- [B5] + // Note: bs.chain contains seals for blocks [F0]...[A2] followed by seals for [final], [B0]...[B5] + bs.Assert().Equal(10, len(bs.assembled.Seals), "unexpected number of seals") + bs.Assert().ElementsMatch(bs.chain[:4], bs.assembled.Seals[:4], "should have included only valid chain of seals") + bs.Assert().ElementsMatch(bs.chain[8:], bs.assembled.Seals[4:], "should have included only valid chain of seals") + + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") +} + +// TestPayloadSeals_EnforceGap checks that builder leaves a 1-block gap between block incorporating the result +// and the block sealing the result. Without this gap, some nodes might not be able to compute the Verifier +// assignment for the seal and therefore reject the block. This edge case only occurs in a very specific situation: +// +// ┌---- [A5] (orphaned fork) +// v +// ...<- [B0] <- [B1] <- [B2] <- [B3] <- [B4{incorporates result R for B1}] <- ░newBlock░ +// +// SCENARIO: +// - block B0 is sealed +// Proposer for ░newBlock░ knows block A5. Hence, it knows a QC for block B4, which contains the Source Of Randomness (SOR) for B4. +// Therefore, the proposer can construct the verifier assignment for [B4{incorporates result R for B1}] +// - Assume that verification was fast enough, so the proposer has sufficient approvals for result R. +// Therefore, the proposer has a candidate seal, sealing result R for block B4, in its mempool. +// +// Replica trying to verify ░newBlock░: +// +// - Assume that the replica does _not_ know A5. Therefore, it _cannot_ compute the verifier assignment for B4. +// +// Problem: If the proposer included the seal for B1, the replica could not check it. +// Solution: There must be a gap between the block incorporating the result (here B4) and +// the block sealing the result. A gap of one block is sufficient. +// +// ┌---- [A5] (orphaned fork) +// v +// ...<- [B0] <- [B1] <- [B2] <- [B3] <- [B4{incorporates result R for B1}] <- [B5] <- [B6{seals B1}] +// ~~~~~~ +// gap +// +// We test the two distinct cases: +// +// (i) Builder does _not_ include seal for B1 when constructing block B5 +// (ii) Builder _includes_ seal for B1 when constructing block B6 +func (bs *BuilderSuite) TestPayloadSeals_EnforceGap() { + // we use bs.parentID as block B0 + b0result := bs.resultForBlock[bs.parentID] + b0seal := unittest.Seal.Fixture(unittest.Seal.WithResult(b0result)) + + // create blocks B1 to B4: + b1 := bs.createAndRecordBlock(bs.blocks[bs.parentID], true) + bchain := unittest.ChainFixtureFrom(3, b1.Header) // creates blocks b2, b3, b4 + b4 := bchain[2] + + // Incorporate result for block B1 into payload of block B4 + resultB1 := bs.resultForBlock[b1.ID()] + receiptB1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultB1)) + b4.SetPayload( + flow.Payload{ + Results: []*flow.ExecutionResult{&receiptB1.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{receiptB1.Meta()}, + }) + + // add blocks B2, B3, B4, A5 to the mocked storage layer (block b0 and b1 are already added): + a5 := unittest.BlockWithParentFixture(b4.Header) + for _, b := range append(bchain, a5) { + bs.storeBlock(b) + } + + // mock for of candidate seal mempool: + bs.pendingSeals = make(map[flow.Identifier]*flow.IncorporatedResultSeal) + b1seal := storeSealForIncorporatedResult(resultB1, b4.ID(), bs.pendingSeals) + + // mock for seals storage layer: + bs.sealDB = &storage.Seals{} + bs.build.seals = bs.sealDB + + bs.T().Run("Build on top of B4 and check that no seals are included", func(t *testing.T) { + bs.sealDB.On("HighestInFork", b4.ID()).Return(b0seal, nil) + + _, err := bs.build.BuildOn(b4.ID(), bs.setter) + require.NoError(t, err) + bs.recPool.AssertExpectations(t) + require.Empty(t, bs.assembled.Seals, "should not include any seals") + }) + + bs.T().Run("Build on top of B5 and check that seals for B1 is included", func(t *testing.T) { + b5 := unittest.BlockWithParentFixture(b4.Header) // creating block b5 + bs.storeBlock(b5) + bs.sealDB.On("HighestInFork", b5.ID()).Return(b0seal, nil) + + _, err := bs.build.BuildOn(b5.ID(), bs.setter) + require.NoError(t, err) + bs.recPool.AssertExpectations(t) + require.Equal(t, 1, len(bs.assembled.Seals), "only seal for B1 expected") + require.Equal(t, b1seal.Seal, bs.assembled.Seals[0]) + }) +} + +// TestPayloadSeals_Duplicates verifies that the builder does not duplicate seals for already sealed blocks: +// +// ... <- [F0] <- [F1] <- [F2] <- [F3] <- [A0] <- [A1] <- [A2] <- [A3] +// +// Where block +// - [F0] ... [F3] sealed blocks but their candidate seals are still included in mempool +// - [A0] ... [A3] unsealed blocks with candidate seals are included in mempool +// +// Expected behaviour: +// - builder should only include seals [A0], ..., [A3] +func (bs *BuilderSuite) TestPayloadSeals_Duplicate() { + // Pretend that the first n blocks are already sealed + n := 4 + lastSeal := bs.chain[n-1] + mockSealDB := &storage.Seals{} + mockSealDB.On("HighestInFork", mock.Anything).Return(lastSeal, nil) + bs.build.seals = mockSealDB + + // seals for all blocks [F0], ..., [A3] are still in the mempool: + bs.pendingSeals = bs.irsMap + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Equal(bs.chain[n:], bs.assembled.Seals, "should have rejected duplicate seals") +} + +// TestPayloadSeals_MissingNextSeal checks how the builder handles the fork +// +// [S] <- [F0] <- [F1] <- [F2] <- [F3] <- [A0] <- [A1] <- [A2] <- [A3] +// +// Where block +// - [S] is sealed and finalized +// - [F0] finalized, unsealed block but _without_ candidate seal in mempool +// - [F1] ... [F3] are finalized, unsealed blocks with candidate seals are included in mempool +// - [A0] ... [A3] non-finalized, unsealed blocks with candidate seals are included in mempool +// +// Expected behaviour: +// - builder should not include any seals as the immediately next seal is not in mempool +func (bs *BuilderSuite) TestPayloadSeals_MissingNextSeal() { + // remove the seal for block [F0] + firstSeal := bs.irsList[0] + delete(bs.irsMap, firstSeal.ID()) + bs.pendingSeals = bs.irsMap + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") + bs.Assert().Empty(bs.assembled.Seals, "should not have included any seals from cutoff chain") +} + +// TestPayloadSeals_MissingInterimSeal checks how the builder handles the fork +// +// [S] <- [F0] <- [F1] <- [F2] <- [F3] <- [A0] <- [A1] <- [A2] <- [A3] +// +// Where block +// - [S] is sealed and finalized +// - [F0] ... [F2] are finalized, unsealed blocks with candidate seals are included in mempool +// - [F4] finalized, unsealed block but _without_ candidate seal in mempool +// - [A0] ... [A3] non-finalized, unsealed blocks with candidate seals are included in mempool +// +// Expected behaviour: +// - builder should only include candidate seals for [F0], [F1], [F2] +func (bs *BuilderSuite) TestPayloadSeals_MissingInterimSeal() { + // remove a seal for block [F4] + seal := bs.irsList[3] + delete(bs.irsMap, seal.ID()) + bs.pendingSeals = bs.irsMap + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") + bs.Assert().ElementsMatch(bs.chain[:3], bs.assembled.Seals, "should have included only beginning of broken chain") +} + +// TestValidatePayloadSeals_ExecutionForks checks how the builder's seal-inclusion logic +// handles execution forks. +// +// we have the chain in storage: +// +// F <- A{Result[F]_1, Result[F]_2, ReceiptMeta[F]_1, ReceiptMeta[F]_2} +// <- B{Result[A]_1, Result[A]_2, ReceiptMeta[A]_1, ReceiptMeta[A]_2} +// <- C{Result[B]_1, Result[B]_2, ReceiptMeta[B]_1, ReceiptMeta[B]_2} +// <- D{Seal for Result[F]_1} +// +// here F is the latest finalized block (with ID bs.finalID) +// +// Note that we are explicitly testing the handling of an execution fork that +// was incorporated _before_ the seal +// +// Blocks: F <----------- A <----------- B +// Results: Result[F]_1 <- Result[A]_1 <- Result[B]_1 :: the root of this execution tree is sealed +// Result[F]_2 <- Result[A]_2 <- Result[B]_2 :: the root of this execution tree conflicts with sealed result +// +// The builder is tasked with creating the payload for block X: +// +// F <- A{..} <- B{..} <- C{..} <- D{..} <- X +// +// We test the two distinct cases: +// +// (i) verify that execution fork conflicting with sealed result is not sealed +// (ii) verify that multiple execution forks are properly handled +func (bs *BuilderSuite) TestValidatePayloadSeals_ExecutionForks() { + bs.build.cfg.expiry = 4 // reduce expiry so collection dedup algorithm doesn't walk past [lastSeal] + + blockF := bs.blocks[bs.finalID] + blocks := []*flow.Block{blockF} + blocks = append(blocks, unittest.ChainFixtureFrom(4, blockF.Header)...) // elements [F, A, B, C, D] + receiptChain1 := unittest.ReceiptChainFor(blocks, unittest.ExecutionResultFixture()) // elements [Result[F]_1, Result[A]_1, Result[B]_1, ...] + receiptChain2 := unittest.ReceiptChainFor(blocks, unittest.ExecutionResultFixture()) // elements [Result[F]_2, Result[A]_2, Result[B]_2, ...] + + for i := 1; i <= 3; i++ { // set payload for blocks A, B, C + blocks[i].SetPayload(flow.Payload{ + Results: []*flow.ExecutionResult{&receiptChain1[i-1].ExecutionResult, &receiptChain2[i-1].ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{receiptChain1[i-1].Meta(), receiptChain2[i-1].Meta()}, + }) + } + sealedResult := receiptChain1[0].ExecutionResult + sealF := unittest.Seal.Fixture(unittest.Seal.WithResult(&sealedResult)) + blocks[4].SetPayload(flow.Payload{ // set payload for block D + Seals: []*flow.Seal{sealF}, + }) + for i := 0; i <= 4; i++ { + // we need to run this several times, as in each iteration as we have _multiple_ execution chains. + // In each iteration, we only mange to reconnect one additional height + unittest.ReconnectBlocksAndReceipts(blocks, receiptChain1) + unittest.ReconnectBlocksAndReceipts(blocks, receiptChain2) + } + + for _, b := range blocks { + bs.storeBlock(b) + } + bs.sealDB = &storage.Seals{} + bs.build.seals = bs.sealDB + bs.sealDB.On("HighestInFork", mock.Anything).Return(sealF, nil) + bs.resultByID[sealedResult.ID()] = &sealedResult + + bs.T().Run("verify that execution fork conflicting with sealed result is not sealed", func(t *testing.T) { + bs.pendingSeals = make(map[flow.Identifier]*flow.IncorporatedResultSeal) + storeSealForIncorporatedResult(&receiptChain2[1].ExecutionResult, blocks[2].ID(), bs.pendingSeals) + + _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter) + require.NoError(t, err) + require.Empty(t, bs.assembled.Seals, "should not have included seal for conflicting execution fork") + }) + + bs.T().Run("verify that multiple execution forks are properly handled", func(t *testing.T) { + bs.pendingSeals = make(map[flow.Identifier]*flow.IncorporatedResultSeal) + sealResultA_1 := storeSealForIncorporatedResult(&receiptChain1[1].ExecutionResult, blocks[2].ID(), bs.pendingSeals) + sealResultB_1 := storeSealForIncorporatedResult(&receiptChain1[2].ExecutionResult, blocks[3].ID(), bs.pendingSeals) + storeSealForIncorporatedResult(&receiptChain2[1].ExecutionResult, blocks[2].ID(), bs.pendingSeals) + storeSealForIncorporatedResult(&receiptChain2[2].ExecutionResult, blocks[3].ID(), bs.pendingSeals) + + _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter) + require.NoError(t, err) + require.ElementsMatch(t, []*flow.Seal{sealResultA_1.Seal, sealResultB_1.Seal}, bs.assembled.Seals, "valid fork should have been sealed") + }) +} + +// TestPayloadReceipts_TraverseExecutionTreeFromLastSealedResult tests the receipt selection: +// Expectation: Builder should trigger ExecutionTree to search Execution Tree from +// last sealed result on respective fork. +// +// We test with the following main chain tree +// +// ┌-[X0] <- [X1{seals ..F4}] +// v +// [lastSeal] <- [F0] <- [F1] <- [F2] <- [F3] <- [F4] <- [A0] <- [A1{seals ..F2}] <- [A2] <- [A3] +// +// Where +// * blocks [lastSeal], [F1], ... [F4], [A0], ... [A4], are created by BuilderSuite +// * latest sealed block for a specific fork is provided by test-local seals storage mock +func (bs *BuilderSuite) TestPayloadReceipts_TraverseExecutionTreeFromLastSealedResult() { + bs.build.cfg.expiry = 4 // reduce expiry so collection dedup algorithm doesn't walk past [lastSeal] + x0 := bs.createAndRecordBlock(bs.blocks[bs.finalID], true) + x1 := bs.createAndRecordBlock(x0, true) + + // set last sealed blocks: + f2 := bs.blocks[bs.finalizedBlockIDs[2]] + f2eal := unittest.Seal.Fixture(unittest.Seal.WithResult(bs.resultForBlock[f2.ID()])) + f4Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(bs.resultForBlock[bs.finalID])) + bs.sealDB = &storage.Seals{} + bs.build.seals = bs.sealDB + + // reset receipts mempool to verify calls made by Builder + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("Size").Return(uint(0)).Maybe() + bs.build.recPool = bs.recPool + + // building on top of X0: latest finalized block in fork is [lastSeal]; expect search to start with sealed result + bs.sealDB.On("HighestInFork", x0.ID()).Return(bs.lastSeal, nil) + bs.recPool.On("ReachableReceipts", bs.lastSeal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() + _, err := bs.build.BuildOn(x0.ID(), bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) + + // building on top of X1: latest finalized block in fork is [F4]; expect search to start with sealed result + bs.sealDB.On("HighestInFork", x1.ID()).Return(f4Seal, nil) + bs.recPool.On("ReachableReceipts", f4Seal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() + _, err = bs.build.BuildOn(x1.ID(), bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) + + // building on top of A3 (with ID bs.parentID): latest finalized block in fork is [F4]; expect search to start with sealed result + bs.sealDB.On("HighestInFork", bs.parentID).Return(f2eal, nil) + bs.recPool.On("ReachableReceipts", f2eal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() + _, err = bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) +} + +// TestPayloadReceipts_IncludeOnlyReceiptsForCurrentFork tests the receipt selection: +// In this test, we check that the Builder provides a BlockFilter which only allows +// blocks on the fork, which we are extending. We construct the following chain tree: +// +// ┌--[X1] ┌-[Y2] ┌-- [A6] +// v v v +// <- [Final] <- [*B1*] <- [*B2*] <- [*B3*] <- [*B4{seals B1}*] <- [*B5*] <- ░newBlock░ +// ^ +// └-- [C3] <- [C4] +// ^--- [D4] +// +// Expectation: BlockFilter should pass blocks marked with star: B1, ... ,B5 +// All other blocks should be filtered out. +// +// Context: +// While the receipt selection itself is performed by the ExecutionTree, the Builder +// controls the selection by providing suitable BlockFilter and ReceiptFilter. +func (bs *BuilderSuite) TestPayloadReceipts_IncludeOnlyReceiptsForCurrentFork() { + b1 := bs.createAndRecordBlock(bs.blocks[bs.finalID], true) + b2 := bs.createAndRecordBlock(b1, true) + b3 := bs.createAndRecordBlock(b2, true) + b4 := bs.createAndRecordBlock(b3, true) + b5 := bs.createAndRecordBlock(b4, true) + + x1 := bs.createAndRecordBlock(bs.blocks[bs.finalID], true) + y2 := bs.createAndRecordBlock(b1, true) + a6 := bs.createAndRecordBlock(b5, true) + + c3 := bs.createAndRecordBlock(b2, true) + c4 := bs.createAndRecordBlock(c3, true) + d4 := bs.createAndRecordBlock(c3, true) + + // set last sealed blocks: + b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(bs.resultForBlock[b1.ID()])) + bs.sealDB = &storage.Seals{} + bs.sealDB.On("HighestInFork", b5.ID()).Return(b1Seal, nil) + bs.build.seals = bs.sealDB + + // setup mock to test the BlockFilter provided by Builder + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("Size").Return(uint(0)).Maybe() + bs.recPool.On("ReachableReceipts", b1Seal.ResultID, mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + blockFilter := args[1].(mempoolAPIs.BlockFilter) + for _, h := range []*flow.Header{b1.Header, b2.Header, b3.Header, b4.Header, b5.Header} { + assert.True(bs.T(), blockFilter(h)) + } + for _, h := range []*flow.Header{bs.blocks[bs.finalID].Header, x1.Header, y2.Header, a6.Header, c3.Header, c4.Header, d4.Header} { + assert.False(bs.T(), blockFilter(h)) + } + }).Return([]*flow.ExecutionReceipt{}, nil).Once() + bs.build.recPool = bs.recPool + + _, err := bs.build.BuildOn(b5.ID(), bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) +} + +// TestPayloadReceipts_SkipDuplicatedReceipts tests the receipt selection: +// Expectation: we check that the Builder provides a ReceiptFilter which +// filters out duplicated receipts. +// Comment: +// While the receipt selection itself is performed by the ExecutionTree, the Builder +// controls the selection by providing suitable BlockFilter and ReceiptFilter. +func (bs *BuilderSuite) TestPayloadReceipts_SkipDuplicatedReceipts() { + // setup mock to test the ReceiptFilter provided by Builder + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("Size").Return(uint(0)).Maybe() + bs.recPool.On("ReachableReceipts", bs.lastSeal.ResultID, mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + receiptFilter := args[2].(mempoolAPIs.ReceiptFilter) + // verify that all receipts already included in blocks are filtered out: + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + assert.False(bs.T(), receiptFilter(rcpt)) + } + } + // Verify that receipts for unsealed blocks, which are _not_ already incorporated are accepted: + for _, block := range bs.blocks { + if block.ID() != bs.firstID { // block with ID bs.firstID is already sealed + rcpt := unittest.ReceiptForBlockFixture(block) + assert.True(bs.T(), receiptFilter(rcpt)) + } + } + }).Return([]*flow.ExecutionReceipt{}, nil).Once() + bs.build.recPool = bs.recPool + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) +} + +// TestPayloadReceipts_SkipReceiptsForSealedBlock tests the receipt selection: +// Expectation: we check that the Builder provides a ReceiptFilter which +// filters out _any_ receipt for the sealed block. +// +// Comment: +// While the receipt selection itself is performed by the ExecutionTree, the Builder +// controls the selection by providing suitable BlockFilter and ReceiptFilter. +func (bs *BuilderSuite) TestPayloadReceipts_SkipReceiptsForSealedBlock() { + // setup mock to test the ReceiptFilter provided by Builder + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("Size").Return(uint(0)).Maybe() + bs.recPool.On("ReachableReceipts", bs.lastSeal.ResultID, mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + receiptFilter := args[2].(mempoolAPIs.ReceiptFilter) + + // receipt for sealed block committing to same result as the sealed result + rcpt := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.firstID])) + assert.False(bs.T(), receiptFilter(rcpt)) + + // receipt for sealed block committing to different result as the sealed result + rcpt = unittest.ReceiptForBlockFixture(bs.blocks[bs.firstID]) + assert.False(bs.T(), receiptFilter(rcpt)) + }).Return([]*flow.ExecutionReceipt{}, nil).Once() + bs.build.recPool = bs.recPool + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.recPool.AssertExpectations(bs.T()) +} + +// TestPayloadReceipts_BlockLimit tests that the builder does not include more +// receipts than the configured maxReceiptCount. +func (bs *BuilderSuite) TestPayloadReceipts_BlockLimit() { + + // Populate the mempool with 5 valid receipts + receipts := []*flow.ExecutionReceipt{} + metas := []*flow.ExecutionReceiptMeta{} + expectedResults := []*flow.ExecutionResult{} + var i uint64 + for i = 0; i < 5; i++ { + blockOnFork := bs.blocks[bs.irsList[i].Seal.BlockID] + pendingReceipt := unittest.ReceiptForBlockFixture(blockOnFork) + receipts = append(receipts, pendingReceipt) + metas = append(metas, pendingReceipt.Meta()) + expectedResults = append(expectedResults, &pendingReceipt.ExecutionResult) + } + bs.pendingReceipts = receipts + + // set maxReceiptCount to 3 + var limit uint = 3 + bs.build.cfg.maxReceiptCount = limit + + // ensure that only 3 of the 5 receipts were included + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(metas[:limit], bs.assembled.Receipts, "should have excluded receipts above maxReceiptCount") + bs.Assert().ElementsMatch(expectedResults[:limit], bs.assembled.Results, "should have excluded results above maxReceiptCount") +} + +// TestPayloadReceipts_AsProvidedByReceiptForest tests the receipt selection. +// Expectation: Builder should embed the Receipts as provided by the ExecutionTree +func (bs *BuilderSuite) TestPayloadReceipts_AsProvidedByReceiptForest() { + var expectedReceipts []*flow.ExecutionReceipt + var expectedMetas []*flow.ExecutionReceiptMeta + var expectedResults []*flow.ExecutionResult + for i := 0; i < 10; i++ { + expectedReceipts = append(expectedReceipts, unittest.ExecutionReceiptFixture()) + expectedMetas = append(expectedMetas, expectedReceipts[i].Meta()) + expectedResults = append(expectedResults, &expectedReceipts[i].ExecutionResult) + } + bs.recPool = &mempool.ExecutionTree{} + bs.recPool.On("Size").Return(uint(0)).Maybe() + bs.recPool.On("AddResult", mock.Anything, mock.Anything).Return(nil).Maybe() + bs.recPool.On("ReachableReceipts", mock.Anything, mock.Anything, mock.Anything).Return(expectedReceipts, nil).Once() + bs.build.recPool = bs.recPool + + _, err := bs.build.BuildOn(bs.parentID, bs.setter) + bs.Require().NoError(err) + bs.Assert().ElementsMatch(expectedMetas, bs.assembled.Receipts, "should include receipts as returned by ExecutionTree") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "should include results as returned by ExecutionTree") + bs.recPool.AssertExpectations(bs.T()) +} + +// TestIntegration_PayloadReceiptNoParentResult is a mini-integration test combining the +// Builder with a full ExecutionTree mempool. We check that the builder does not include +// receipts whose PreviousResult is not already incorporated in the chain. +// +// Here we create 4 consecutive blocks S, A, B, and C, where A contains a valid +// receipt for block S, but blocks B and C have empty payloads. +// +// We populate the mempool with valid receipts for blocks A, and C, but NOT for +// block B. +// +// The expected behaviour is that the builder should not include the receipt for +// block C, because the chain and the mempool do not contain a valid receipt for +// the parent result (block B's result). +// +// ... <- S[ER{parent}] <- A[ER{S}] <- B <- C <- X (candidate) +func (bs *BuilderSuite) TestIntegration_PayloadReceiptNoParentResult() { + // make blocks S, A, B, C + parentReceipt := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + blockSABC := unittest.ChainFixtureFrom(4, bs.blocks[bs.parentID].Header) + resultS := unittest.ExecutionResultFixture(unittest.WithBlock(blockSABC[0]), unittest.WithPreviousResult(*bs.resultForBlock[bs.parentID])) + receiptSABC := unittest.ReceiptChainFor(blockSABC, resultS) + blockSABC[0].Payload.Receipts = []*flow.ExecutionReceiptMeta{parentReceipt.Meta()} + blockSABC[0].Payload.Results = []*flow.ExecutionResult{&parentReceipt.ExecutionResult} + blockSABC[1].Payload.Receipts = []*flow.ExecutionReceiptMeta{receiptSABC[0].Meta()} + blockSABC[1].Payload.Results = []*flow.ExecutionResult{&receiptSABC[0].ExecutionResult} + blockSABC[2].Payload.Receipts = []*flow.ExecutionReceiptMeta{} + blockSABC[3].Payload.Receipts = []*flow.ExecutionReceiptMeta{} + unittest.ReconnectBlocksAndReceipts(blockSABC, receiptSABC) // update block header so that blocks are chained together + + bs.storeBlock(blockSABC[0]) + bs.storeBlock(blockSABC[1]) + bs.storeBlock(blockSABC[2]) + bs.storeBlock(blockSABC[3]) + + // Instantiate real Execution Tree mempool; + bs.build.recPool = mempoolImpl.NewExecutionTree() + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + _, err := bs.build.recPool.AddReceipt(rcpt, bs.blocks[rcpt.ExecutionResult.BlockID].Header) + bs.NoError(err) + } + } + // for receipts _not_ included in blocks, add only receipt for A and C but NOT B + _, _ = bs.build.recPool.AddReceipt(receiptSABC[1], blockSABC[1].Header) + _, _ = bs.build.recPool.AddReceipt(receiptSABC[3], blockSABC[3].Header) + + _, err := bs.build.BuildOn(blockSABC[3].ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := flow.ExecutionReceiptMetaList{receiptSABC[1].Meta()} + expectedResults := flow.ExecutionResultList{&receiptSABC[1].ExecutionResult} + bs.Assert().Equal(expectedReceipts, bs.assembled.Receipts, "payload should contain only receipt for block a") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "payload should contain only result for block a") +} + +// TestIntegration_ExtendDifferentExecutionPathsOnSameFork tests that the +// builder includes receipts that form different valid execution paths contained +// on the current fork. +// +// candidate +// P <- A[ER{P}] <- B[ER{A}, ER{A}'] <- X[ER{B}, ER{B}'] +func (bs *BuilderSuite) TestIntegration_ExtendDifferentExecutionPathsOnSameFork() { + + // A is a block containing a valid receipt for block P + recP := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + A := unittest.BlockWithParentFixture(bs.headers[bs.parentID]) + A.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recP.Meta()}, + Results: []*flow.ExecutionResult{&recP.ExecutionResult}, + }) + + // B is a block containing two valid receipts, with different results, for + // block A + resA1 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA1)) + resA2 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA2)) + B := unittest.BlockWithParentFixture(A.Header) + B.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recA1.Meta(), recA2.Meta()}, + Results: []*flow.ExecutionResult{&recA1.ExecutionResult, &recA2.ExecutionResult}, + }) + + bs.storeBlock(A) + bs.storeBlock(B) + + // Instantiate real Execution Tree mempool; + bs.build.recPool = mempoolImpl.NewExecutionTree() + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + _, err := bs.build.recPool.AddReceipt(rcpt, bs.blocks[rcpt.ExecutionResult.BlockID].Header) + bs.NoError(err) + } + } + + // Create two valid receipts for block B which build on different receipts + // for the parent block (A); recB1 builds on top of RecA1, whilst recB2 + // builds on top of RecA2. + resB1 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA1.ExecutionResult)) + recB1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB1)) + resB2 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA2.ExecutionResult)) + recB2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB2)) + + // Add recB1 and recB2 to the mempool for inclusion in the next candidate + _, _ = bs.build.recPool.AddReceipt(recB1, B.Header) + _, _ = bs.build.recPool.AddReceipt(recB2, B.Header) + + _, err := bs.build.BuildOn(B.ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := flow.ExecutionReceiptMetaList{recB1.Meta(), recB2.Meta()} + expectedResults := flow.ExecutionResultList{&recB1.ExecutionResult, &recB2.ExecutionResult} + bs.Assert().Equal(expectedReceipts, bs.assembled.Receipts, "payload should contain receipts from valid execution forks") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "payload should contain results from valid execution forks") +} + +// TestIntegration_ExtendDifferentExecutionPathsOnDifferentForks tests that the +// builder picks up receipts that were already included in a different fork. +// +// candidate +// P <- A[ER{P}] <- B[ER{A}] <- X[ER{A}',ER{B}, ER{B}'] +// | +// < ------ C[ER{A}'] +// +// Where: +// - ER{A} and ER{A}' are receipts for block A that don't have the same +// result. +// - ER{B} is a receipt for B with parent result ER{A} +// - ER{B}' is a receipt for B with parent result ER{A}' +// +// When buiding on top of B, we expect the candidate payload to contain ER{A}', +// ER{B}, and ER{B}' +// +// ER{P} <- ER{A} <- ER{B} +// | +// < ER{A}' <- ER{B}' +func (bs *BuilderSuite) TestIntegration_ExtendDifferentExecutionPathsOnDifferentForks() { + // A is a block containing a valid receipt for block P + recP := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + A := unittest.BlockWithParentFixture(bs.headers[bs.parentID]) + A.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recP.Meta()}, + Results: []*flow.ExecutionResult{&recP.ExecutionResult}, + }) + + // B is a block that builds on A containing a valid receipt for A + resA1 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA1)) + B := unittest.BlockWithParentFixture(A.Header) + B.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recA1.Meta()}, + Results: []*flow.ExecutionResult{&recA1.ExecutionResult}, + }) + + // C is another block that builds on A containing a valid receipt for A but + // different from the receipt contained in B + resA2 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA2)) + C := unittest.BlockWithParentFixture(A.Header) + C.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recA2.Meta()}, + Results: []*flow.ExecutionResult{&recA2.ExecutionResult}, + }) + + bs.storeBlock(A) + bs.storeBlock(B) + bs.storeBlock(C) + + // Instantiate real Execution Tree mempool; + bs.build.recPool = mempoolImpl.NewExecutionTree() + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + _, err := bs.build.recPool.AddReceipt(rcpt, bs.blocks[rcpt.ExecutionResult.BlockID].Header) + bs.NoError(err) + } + } + + // create and add a receipt for block B which builds on top of recA2, which + // is not on the same execution fork + resB1 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA1.ExecutionResult)) + recB1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB1)) + resB2 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA2.ExecutionResult)) + recB2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB2)) + + _, err := bs.build.recPool.AddReceipt(recB1, B.Header) + bs.Require().NoError(err) + _, err = bs.build.recPool.AddReceipt(recB2, B.Header) + bs.Require().NoError(err) + + _, err = bs.build.BuildOn(B.ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := []*flow.ExecutionReceiptMeta{recA2.Meta(), recB1.Meta(), recB2.Meta()} + expectedResults := []*flow.ExecutionResult{&recA2.ExecutionResult, &recB1.ExecutionResult, &recB2.ExecutionResult} + bs.Assert().ElementsMatch(expectedReceipts, bs.assembled.Receipts, "builder should extend different execution paths") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "builder should extend different execution paths") +} + +// TestIntegration_DuplicateReceipts checks that the builder does not re-include +// receipts that are already incorporated in blocks on the fork. +// +// P <- A(r_P) <- B(r_A) <- X (candidate) +func (bs *BuilderSuite) TestIntegration_DuplicateReceipts() { + // A is a block containing a valid receipt for block P + recP := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + A := unittest.BlockWithParentFixture(bs.headers[bs.parentID]) + A.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recP.Meta()}, + Results: []*flow.ExecutionResult{&recP.ExecutionResult}, + }) + + // B is a block that builds on A containing a valid receipt for A + resA1 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA1)) + B := unittest.BlockWithParentFixture(A.Header) + B.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recA1.Meta()}, + Results: []*flow.ExecutionResult{&recA1.ExecutionResult}, + }) + + bs.storeBlock(A) + bs.storeBlock(B) + + // Instantiate real Execution Tree mempool; + bs.build.recPool = mempoolImpl.NewExecutionTree() + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + _, err := bs.build.recPool.AddReceipt(rcpt, bs.blocks[rcpt.ExecutionResult.BlockID].Header) + bs.NoError(err) + } + } + + _, err := bs.build.BuildOn(B.ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := []*flow.ExecutionReceiptMeta{} + expectedResults := []*flow.ExecutionResult{} + bs.Assert().ElementsMatch(expectedReceipts, bs.assembled.Receipts, "builder should not include receipts that are already incorporated in the current fork") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "builder should not include results that were already incorporated") +} + +// TestIntegration_ResultAlreadyIncorporated checks that the builder includes +// receipts for results that were already incorporated in blocks on the fork. +// +// P <- A(ER[P]) <- X (candidate) +func (bs *BuilderSuite) TestIntegration_ResultAlreadyIncorporated() { + // A is a block containing a valid receipt for block P + recP := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + A := unittest.BlockWithParentFixture(bs.headers[bs.parentID]) + A.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recP.Meta()}, + Results: []*flow.ExecutionResult{&recP.ExecutionResult}, + }) + + recP_B := unittest.ExecutionReceiptFixture(unittest.WithResult(&recP.ExecutionResult)) + + bs.storeBlock(A) + + // Instantiate real Execution Tree mempool; + bs.build.recPool = mempoolImpl.NewExecutionTree() + for _, block := range bs.blocks { + resultByID := block.Payload.Results.Lookup() + for _, meta := range block.Payload.Receipts { + result := resultByID[meta.ResultID] + rcpt := flow.ExecutionReceiptFromMeta(*meta, *result) + _, err := bs.build.recPool.AddReceipt(rcpt, bs.blocks[rcpt.ExecutionResult.BlockID].Header) + bs.NoError(err) + } + } + + _, err := bs.build.recPool.AddReceipt(recP_B, bs.blocks[recP_B.ExecutionResult.BlockID].Header) + bs.NoError(err) + + _, err = bs.build.BuildOn(A.ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := []*flow.ExecutionReceiptMeta{recP_B.Meta()} + expectedResults := []*flow.ExecutionResult{} + bs.Assert().ElementsMatch(expectedReceipts, bs.assembled.Receipts, "builder should include receipt metas for results that were already incorporated") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "builder should not include results that were already incorporated") +} + +func storeSealForIncorporatedResult(result *flow.ExecutionResult, incorporatingBlockID flow.Identifier, pendingSeals map[flow.Identifier]*flow.IncorporatedResultSeal) *flow.IncorporatedResultSeal { + incorporatedResultSeal := unittest.IncorporatedResultSeal.Fixture( + unittest.IncorporatedResultSeal.WithResult(result), + unittest.IncorporatedResultSeal.WithIncorporatedBlockID(incorporatingBlockID), + ) + pendingSeals[incorporatedResultSeal.ID()] = incorporatedResultSeal + return incorporatedResultSeal +} + +// TestIntegration_RepopulateExecutionTreeAtStartup tests that the +// builder includes receipts for candidate block after fresh start, meaning +// it will repopulate execution tree in constructor +// +// P <- A[ER{P}] <- B[ER{A}, ER{A}'] <- C <- X[ER{B}, ER{B}', ER{C} ] +// | +// finalized +func (bs *BuilderSuite) TestIntegration_RepopulateExecutionTreeAtStartup() { + // setup initial state + // A is a block containing a valid receipt for block P + recP := unittest.ExecutionReceiptFixture(unittest.WithResult(bs.resultForBlock[bs.parentID])) + A := unittest.BlockWithParentFixture(bs.headers[bs.parentID]) + A.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recP.Meta()}, + Results: []*flow.ExecutionResult{&recP.ExecutionResult}, + }) + + // B is a block containing two valid receipts, with different results, for + // block A + resA1 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA1)) + resA2 := unittest.ExecutionResultFixture(unittest.WithBlock(A), unittest.WithPreviousResult(recP.ExecutionResult)) + recA2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resA2)) + B := unittest.BlockWithParentFixture(A.Header) + B.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{recA1.Meta(), recA2.Meta()}, + Results: []*flow.ExecutionResult{&recA1.ExecutionResult, &recA2.ExecutionResult}, + }) + + C := unittest.BlockWithParentFixture(B.Header) + + bs.storeBlock(A) + bs.storeBlock(B) + bs.storeBlock(C) + + // store execution results + for _, block := range []*flow.Block{A, B, C} { + // for current block create empty receipts list + bs.receiptsByBlockID[block.ID()] = flow.ExecutionReceiptList{} + + for _, result := range block.Payload.Results { + bs.resultByID[result.ID()] = result + } + for _, meta := range block.Payload.Receipts { + receipt := flow.ExecutionReceiptFromMeta(*meta, *bs.resultByID[meta.ResultID]) + bs.receiptsByID[meta.ID()] = receipt + bs.receiptsByBlockID[receipt.ExecutionResult.BlockID] = append(bs.receiptsByBlockID[receipt.ExecutionResult.BlockID], receipt) + } + } + + // mark A as finalized + bs.finalID = A.ID() + + // set up no-op dependencies + noopMetrics := metrics.NewNoopCollector() + noopTracer := trace.NewNoopTracer() + + // Instantiate real Execution Tree mempool; + recPool := mempoolImpl.NewExecutionTree() + + // create builder which has to repopulate execution tree + var err error + bs.build, err = NewBuilder( + noopMetrics, + bs.db, + bs.state, + bs.headerDB, + bs.sealDB, + bs.indexDB, + bs.blockDB, + bs.resultDB, + bs.receiptsDB, + bs.guarPool, + bs.sealPool, + recPool, + noopTracer, + ) + require.NoError(bs.T(), err) + bs.build.cfg.expiry = 11 + + // Create two valid receipts for block B which build on different receipts + // for the parent block (A); recB1 builds on top of RecA1, whilst recB2 + // builds on top of RecA2. + resB1 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA1.ExecutionResult)) + recB1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB1)) + resB2 := unittest.ExecutionResultFixture(unittest.WithBlock(B), unittest.WithPreviousResult(recA2.ExecutionResult)) + recB2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resB2)) + resC := unittest.ExecutionResultFixture(unittest.WithBlock(C), unittest.WithPreviousResult(recB1.ExecutionResult)) + recC := unittest.ExecutionReceiptFixture(unittest.WithResult(resC)) + + // Add recB1 and recB2 to the mempool for inclusion in the next candidate + _, _ = bs.build.recPool.AddReceipt(recB1, B.Header) + _, _ = bs.build.recPool.AddReceipt(recB2, B.Header) + _, _ = bs.build.recPool.AddReceipt(recC, C.Header) + + _, err = bs.build.BuildOn(C.ID(), bs.setter) + bs.Require().NoError(err) + expectedReceipts := flow.ExecutionReceiptMetaList{recB1.Meta(), recB2.Meta(), recC.Meta()} + expectedResults := flow.ExecutionResultList{&recB1.ExecutionResult, &recB2.ExecutionResult, &recC.ExecutionResult} + bs.Assert().ElementsMatch(expectedReceipts, bs.assembled.Receipts, "payload should contain receipts from valid execution forks") + bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "payload should contain results from valid execution forks") +} diff --git a/module/finalizer/collection/finalizer_pebble.go b/module/finalizer/collection/finalizer_pebble.go new file mode 100644 index 00000000000..bfe1d76ae4f --- /dev/null +++ b/module/finalizer/collection/finalizer_pebble.go @@ -0,0 +1,176 @@ +package collection + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" +) + +// Finalizer is a simple wrapper around our temporary state to clean up after a +// block has been finalized. This involves removing the transactions within the +// finalized collection from the mempool and updating the finalized boundary in +// the cluster state. +type Finalizer struct { + db *badger.DB + transactions mempool.Transactions + prov network.Engine + metrics module.CollectionMetrics +} + +// NewFinalizer creates a new finalizer for collection nodes. +func NewFinalizer( + db *badger.DB, + transactions mempool.Transactions, + prov network.Engine, + metrics module.CollectionMetrics, +) *Finalizer { + f := &Finalizer{ + db: db, + transactions: transactions, + prov: prov, + metrics: metrics, + } + return f +} + +// MakeFinal handles finalization logic for a block. +// +// The newly finalized block, and all un-finalized ancestors, are marked as +// finalized in the cluster state. All transactions included in the collections +// within the finalized blocks are removed from the mempool. +// +// This assumes that transactions are added to persistent state when they are +// included in a block proposal. Between entering the non-finalized chain state +// and being finalized, entities should be present in both the volatile memory +// pools and persistent storage. +// No errors are expected during normal operation. +func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { + return operation.RetryOnConflict(f.db.Update, func(tx *badger.Txn) error { + + // retrieve the header of the block we want to finalize + var header flow.Header + err := operation.RetrieveHeader(blockID, &header)(tx) + if err != nil { + return fmt.Errorf("could not retrieve header: %w", err) + } + + // retrieve the current finalized cluster state boundary + var boundary uint64 + err = operation.RetrieveClusterFinalizedHeight(header.ChainID, &boundary)(tx) + if err != nil { + return fmt.Errorf("could not retrieve boundary: %w", err) + } + + // retrieve the ID of the last finalized block as marker for stopping + var headID flow.Identifier + err = operation.LookupClusterBlockHeight(header.ChainID, boundary, &headID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve head: %w", err) + } + + // there are no blocks to finalize, we may have already finalized + // this block - exit early + if boundary >= header.Height { + return nil + } + + // To finalize all blocks from the currently finalized one up to and + // including the current, we first enumerate each of these blocks. + // We start at the youngest block and remember all visited blocks, + // while tracing back until we reach the finalized state + steps := []*flow.Header{&header} + parentID := header.ParentID + for parentID != headID { + var parent flow.Header + err = operation.RetrieveHeader(parentID, &parent)(tx) + if err != nil { + return fmt.Errorf("could not retrieve parent (%x): %w", parentID, err) + } + steps = append(steps, &parent) + parentID = parent.ParentID + } + + // now we can step backwards in order to go from oldest to youngest; for + // each header, we reconstruct the block and then apply the related + // changes to the protocol state + for i := len(steps) - 1; i >= 0; i-- { + clusterBlockID := steps[i].ID() + + // look up the transactions included in the payload + step := steps[i] + var payload cluster.Payload + err = procedure.RetrieveClusterPayload(clusterBlockID, &payload)(tx) + if err != nil { + return fmt.Errorf("could not retrieve payload for cluster block (id=%x): %w", clusterBlockID, err) + } + + // remove the transactions from the memory pool + for _, colTx := range payload.Collection.Transactions { + txID := colTx.ID() + // ignore result -- we don't care whether the transaction was in the pool + _ = f.transactions.Remove(txID) + } + + // finalize the block in cluster state + err = procedure.FinalizeClusterBlock(clusterBlockID)(tx) + if err != nil { + return fmt.Errorf("could not finalize cluster block (id=%x): %w", clusterBlockID, err) + } + + block := &cluster.Block{ + Header: step, + Payload: &payload, + } + f.metrics.ClusterBlockFinalized(block) + + // if the finalized collection is empty, we don't need to include it + // in the reference height index or submit it to consensus nodes + if len(payload.Collection.Transactions) == 0 { + continue + } + + // look up the reference block height to populate index + var refBlock flow.Header + err = operation.RetrieveHeader(payload.ReferenceBlockID, &refBlock)(tx) + if err != nil { + return fmt.Errorf("could not retrieve reference block (id=%x): %w", payload.ReferenceBlockID, err) + } + // index the finalized cluster block by reference block height + err = operation.IndexClusterBlockByReferenceHeight(refBlock.Height, clusterBlockID)(tx) + if err != nil { + return fmt.Errorf("could not index cluster block (id=%x) by reference height (%d): %w", clusterBlockID, refBlock.Height, err) + } + + //TODO when we incorporate HotStuff AND require BFT, the consensus + // node will need to be able ensure finalization by checking a + // 3-chain of children for this block. Probably it will be simplest + // to have a follower engine configured for the cluster chain + // running on consensus nodes, rather than pushing finalized blocks + // explicitly. + // For now, we just use the parent signers as the guarantors of this + // collection. + + // TODO add real signatures here (2711) + f.prov.SubmitLocal(&messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: payload.Collection.ID(), + ReferenceBlockID: payload.ReferenceBlockID, + ChainID: header.ChainID, + SignerIndices: step.ParentVoterIndices, + Signature: nil, // TODO: to remove because it's not easily verifiable by consensus nodes + }, + }) + } + + return nil + }) +} diff --git a/module/finalizer/collection/finalizer_pebble_test.go b/module/finalizer/collection/finalizer_pebble_test.go new file mode 100644 index 00000000000..fa92d3eeafe --- /dev/null +++ b/module/finalizer/collection/finalizer_pebble_test.go @@ -0,0 +1,374 @@ +package collection_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + model "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/finalizer/collection" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/mocknetwork" + cluster "github.com/onflow/flow-go/state/cluster/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestFinalizer(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + // reference block on the main consensus chain + refBlock := unittest.BlockHeaderFixture() + // genesis block for the cluster chain + genesis := model.Genesis() + + metrics := metrics.NewNoopCollector() + + var state *cluster.State + + pool := herocache.NewTransactions(1000, unittest.Logger(), metrics) + + // a helper function to clean up shared state between tests + cleanup := func() { + // wipe the DB + err := db.DropAll() + require.Nil(t, err) + // clear the mempool + for _, tx := range pool.All() { + pool.Remove(tx.ID()) + } + } + + // a helper function to bootstrap with the genesis block + bootstrap := func() { + stateRoot, err := cluster.NewStateRoot(genesis, unittest.QuorumCertificateFixture(), 0) + require.NoError(t, err) + state, err = cluster.Bootstrap(db, stateRoot) + require.NoError(t, err) + err = db.Update(operation.InsertHeader(refBlock.ID(), refBlock)) + require.NoError(t, err) + } + + // a helper function to insert a block + insert := func(block model.Block) { + err := db.Update(procedure.InsertClusterBlock(&block)) + assert.Nil(t, err) + } + + t.Run("non-existent block", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + fakeBlockID := unittest.IdentifierFixture() + err := finalizer.MakeFinal(fakeBlockID) + assert.Error(t, err) + }) + + t.Run("already finalized block", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // tx1 is included in the finalized block + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 1 }) + assert.True(t, pool.Add(&tx1)) + + // create a new block on genesis + block := unittest.ClusterBlockWithParent(genesis) + block.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx1)) + insert(block) + + // finalize the block + err := finalizer.MakeFinal(block.ID()) + assert.Nil(t, err) + + // finalize the block again - this should be a no-op + err = finalizer.MakeFinal(block.ID()) + assert.Nil(t, err) + }) + + t.Run("unconnected block", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // create a new block that isn't connected to a parent + block := unittest.ClusterBlockWithParent(genesis) + block.Header.ParentID = unittest.IdentifierFixture() + block.SetPayload(model.EmptyPayload(refBlock.ID())) + insert(block) + + // try to finalize - this should fail + err := finalizer.MakeFinal(block.ID()) + assert.Error(t, err) + }) + + t.Run("empty collection block", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // create a block with empty payload on genesis + block := unittest.ClusterBlockWithParent(genesis) + block.SetPayload(model.EmptyPayload(refBlock.ID())) + insert(block) + + // finalize the block + err := finalizer.MakeFinal(block.ID()) + assert.Nil(t, err) + + // check finalized boundary using cluster state + final, err := state.Final().Head() + assert.Nil(t, err) + assert.Equal(t, block.ID(), final.ID()) + + // collection should not have been propagated + prov.AssertNotCalled(t, "SubmitLocal", mock.Anything) + }) + + t.Run("finalize single block", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // tx1 is included in the finalized block and mempool + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 1 }) + assert.True(t, pool.Add(&tx1)) + // tx2 is only in the mempool + tx2 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 2 }) + assert.True(t, pool.Add(&tx2)) + + // create a block containing tx1 on top of genesis + block := unittest.ClusterBlockWithParent(genesis) + block.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx1)) + insert(block) + + // finalize the block + err := finalizer.MakeFinal(block.ID()) + assert.Nil(t, err) + + // tx1 should have been removed from mempool + assert.False(t, pool.Has(tx1.ID())) + // tx2 should still be in mempool + assert.True(t, pool.Has(tx2.ID())) + + // check finalized boundary using cluster state + final, err := state.Final().Head() + assert.Nil(t, err) + assert.Equal(t, block.ID(), final.ID()) + assertClusterBlocksIndexedByReferenceHeight(t, db, refBlock.Height, final.ID()) + + // block should be passed to provider + prov.AssertNumberOfCalls(t, "SubmitLocal", 1) + prov.AssertCalled(t, "SubmitLocal", &messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: block.Payload.Collection.ID(), + ReferenceBlockID: refBlock.ID(), + ChainID: block.Header.ChainID, + SignerIndices: block.Header.ParentVoterIndices, + Signature: nil, + }, + }) + }) + + // when finalizing a block with un-finalized ancestors, those ancestors should be finalized as well + t.Run("finalize multiple blocks together", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // tx1 is included in the first finalized block and mempool + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 1 }) + assert.True(t, pool.Add(&tx1)) + // tx2 is included in the second finalized block and mempool + tx2 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 2 }) + assert.True(t, pool.Add(&tx2)) + + // create a block containing tx1 on top of genesis + block1 := unittest.ClusterBlockWithParent(genesis) + block1.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx1)) + insert(block1) + + // create a block containing tx2 on top of block1 + block2 := unittest.ClusterBlockWithParent(&block1) + block2.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx2)) + insert(block2) + + // finalize block2 (should indirectly finalize block1 as well) + err := finalizer.MakeFinal(block2.ID()) + assert.Nil(t, err) + + // tx1 and tx2 should have been removed from mempool + assert.False(t, pool.Has(tx1.ID())) + assert.False(t, pool.Has(tx2.ID())) + + // check finalized boundary using cluster state + final, err := state.Final().Head() + assert.Nil(t, err) + assert.Equal(t, block2.ID(), final.ID()) + assertClusterBlocksIndexedByReferenceHeight(t, db, refBlock.Height, block1.ID(), block2.ID()) + + // both blocks should be passed to provider + prov.AssertNumberOfCalls(t, "SubmitLocal", 2) + prov.AssertCalled(t, "SubmitLocal", &messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: block1.Payload.Collection.ID(), + ReferenceBlockID: refBlock.ID(), + ChainID: block1.Header.ChainID, + SignerIndices: block1.Header.ParentVoterIndices, + Signature: nil, + }, + }) + prov.AssertCalled(t, "SubmitLocal", &messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: block2.Payload.Collection.ID(), + ReferenceBlockID: refBlock.ID(), + ChainID: block2.Header.ChainID, + SignerIndices: block2.Header.ParentVoterIndices, + Signature: nil, + }, + }) + }) + + t.Run("finalize with un-finalized child", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // tx1 is included in the finalized parent block and mempool + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 1 }) + assert.True(t, pool.Add(&tx1)) + // tx2 is included in the un-finalized block and mempool + tx2 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 2 }) + assert.True(t, pool.Add(&tx2)) + + // create a block containing tx1 on top of genesis + block1 := unittest.ClusterBlockWithParent(genesis) + block1.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx1)) + insert(block1) + + // create a block containing tx2 on top of block1 + block2 := unittest.ClusterBlockWithParent(&block1) + block2.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx2)) + insert(block2) + + // finalize block1 (should NOT finalize block2) + err := finalizer.MakeFinal(block1.ID()) + assert.Nil(t, err) + + // tx1 should have been removed from mempool + assert.False(t, pool.Has(tx1.ID())) + // tx2 should NOT have been removed from mempool (since block2 wasn't finalized) + assert.True(t, pool.Has(tx2.ID())) + + // check finalized boundary using cluster state + final, err := state.Final().Head() + assert.Nil(t, err) + assert.Equal(t, block1.ID(), final.ID()) + assertClusterBlocksIndexedByReferenceHeight(t, db, refBlock.Height, block1.ID()) + + // block should be passed to provider + prov.AssertNumberOfCalls(t, "SubmitLocal", 1) + prov.AssertCalled(t, "SubmitLocal", &messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: block1.Payload.Collection.ID(), + ReferenceBlockID: refBlock.ID(), + ChainID: block1.Header.ChainID, + SignerIndices: block1.Header.ParentVoterIndices, + Signature: nil, + }, + }) + }) + + // when finalizing a block with a conflicting fork, the fork should not be finalized. + t.Run("conflicting fork", func(t *testing.T) { + bootstrap() + defer cleanup() + + prov := new(mocknetwork.Engine) + prov.On("SubmitLocal", mock.Anything) + finalizer := collection.NewFinalizer(db, pool, prov, metrics) + + // tx1 is included in the finalized block and mempool + tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 1 }) + assert.True(t, pool.Add(&tx1)) + // tx2 is included in the conflicting block and mempool + tx2 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { tx.ProposalKey.SequenceNumber = 2 }) + assert.True(t, pool.Add(&tx2)) + + // create a block containing tx1 on top of genesis + block1 := unittest.ClusterBlockWithParent(genesis) + block1.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx1)) + insert(block1) + + // create a block containing tx2 on top of genesis (conflicting with block1) + block2 := unittest.ClusterBlockWithParent(genesis) + block2.SetPayload(model.PayloadFromTransactions(refBlock.ID(), &tx2)) + insert(block2) + + // finalize block1 + err := finalizer.MakeFinal(block1.ID()) + assert.Nil(t, err) + + // tx1 should have been removed from mempool + assert.False(t, pool.Has(tx1.ID())) + // tx2 should NOT have been removed from mempool (since block2 wasn't finalized) + assert.True(t, pool.Has(tx2.ID())) + + // check finalized boundary using cluster state + final, err := state.Final().Head() + assert.Nil(t, err) + assert.Equal(t, block1.ID(), final.ID()) + assertClusterBlocksIndexedByReferenceHeight(t, db, refBlock.Height, block1.ID()) + + // block should be passed to provider + prov.AssertNumberOfCalls(t, "SubmitLocal", 1) + prov.AssertCalled(t, "SubmitLocal", &messages.SubmitCollectionGuarantee{ + Guarantee: flow.CollectionGuarantee{ + CollectionID: block1.Payload.Collection.ID(), + ReferenceBlockID: refBlock.ID(), + ChainID: block1.Header.ChainID, + SignerIndices: block1.Header.ParentVoterIndices, + Signature: nil, + }, + }) + }) + }) +} + +// assertClusterBlocksIndexedByReferenceHeight checks the given cluster blocks have +// been indexed by the given reference block height, which is expected as part of +// finalization. +func assertClusterBlocksIndexedByReferenceHeight(t *testing.T, db *badger.DB, refHeight uint64, clusterBlockIDs ...flow.Identifier) { + var ids []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(refHeight, refHeight, &ids)) + require.NoError(t, err) + assert.ElementsMatch(t, clusterBlockIDs, ids) +} diff --git a/module/finalizer/consensus/finalizer_pebble.go b/module/finalizer/consensus/finalizer_pebble.go new file mode 100644 index 00000000000..b5fd97de564 --- /dev/null +++ b/module/finalizer/consensus/finalizer_pebble.go @@ -0,0 +1,129 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package consensus + +import ( + "context" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// Finalizer is a simple wrapper around our temporary state to clean up after a +// block has been fully finalized to the persistent protocol state. +type Finalizer struct { + db *badger.DB + headers storage.Headers + state protocol.FollowerState + cleanup CleanupFunc + tracer module.Tracer +} + +// NewFinalizer creates a new finalizer for the temporary state. +func NewFinalizer(db *badger.DB, + headers storage.Headers, + state protocol.FollowerState, + tracer module.Tracer, + options ...func(*Finalizer)) *Finalizer { + f := &Finalizer{ + db: db, + state: state, + headers: headers, + cleanup: CleanupNothing(), + tracer: tracer, + } + for _, option := range options { + option(f) + } + return f +} + +// MakeFinal will finalize the block with the given ID and clean up the memory +// pools after it. +// +// This assumes that guarantees and seals are already in persistent state when +// included in a block proposal. Between entering the non-finalized chain state +// and being finalized, entities should be present in both the volatile memory +// pools and persistent storage. +// No errors are expected during normal operation. +func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { + + span, ctx := f.tracer.StartBlockSpan(context.Background(), blockID, trace.CONFinalizerFinalizeBlock) + defer span.End() + + // STEP ONE: This is an idempotent operation. In case we are trying to + // finalize a block that is already below finalized height, we want to do + // one of two things: if it conflicts with the block already finalized at + // that height, it's an invalid operation. Otherwise, it is a no-op. + + var finalized uint64 + err := f.db.View(operation.RetrieveFinalizedHeight(&finalized)) + if err != nil { + return fmt.Errorf("could not retrieve finalized height: %w", err) + } + + pending, err := f.headers.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve pending header: %w", err) + } + + if pending.Height <= finalized { + dupID, err := f.headers.BlockIDByHeight(pending.Height) + if err != nil { + return fmt.Errorf("could not retrieve finalized equivalent: %w", err) + } + if dupID != blockID { + return fmt.Errorf("cannot finalize pending block conflicting with finalized state (height: %d, pending: %x, finalized: %x)", pending.Height, blockID, dupID) + } + return nil + } + + // STEP TWO: At least one block in the chain back to the finalized state is + // a valid candidate for finalization. Figure out all blocks between the + // to-be-finalized block and the last finalized block. If we can't trace + // back to the last finalized block, this is also an invalid call. + + var finalID flow.Identifier + err = f.db.View(operation.LookupBlockHeight(finalized, &finalID)) + if err != nil { + return fmt.Errorf("could not retrieve finalized header: %w", err) + } + pendingIDs := []flow.Identifier{blockID} + ancestorID := pending.ParentID + for ancestorID != finalID { + ancestor, err := f.headers.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not retrieve parent (%x): %w", ancestorID, err) + } + if ancestor.Height < finalized { + return fmt.Errorf("cannot finalize pending block unconnected to last finalized block (height: %d, finalized: %d)", ancestor.Height, finalized) + } + pendingIDs = append(pendingIDs, ancestorID) + ancestorID = ancestor.ParentID + } + + // STEP THREE: We walk backwards through the collected ancestors, starting + // with the first block after finalizing state, and finalize them one by + // one in the protocol state. + + for i := len(pendingIDs) - 1; i >= 0; i-- { + pendingID := pendingIDs[i] + err = f.state.Finalize(ctx, pendingID) + if err != nil { + return fmt.Errorf("could not finalize block (%x): %w", pendingID, err) + } + err := f.cleanup(pendingID) + if err != nil { + return fmt.Errorf("could not execute cleanup (%x): %w", pendingID, err) + } + } + + return nil +} diff --git a/module/finalizer/consensus/finalizer_pebble_test.go b/module/finalizer/consensus/finalizer_pebble_test.go new file mode 100644 index 00000000000..35b20705ec4 --- /dev/null +++ b/module/finalizer/consensus/finalizer_pebble_test.go @@ -0,0 +1,219 @@ +package consensus + +import ( + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/trace" + mockprot "github.com/onflow/flow-go/state/protocol/mock" + storage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + mockstor "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +func LogCleanup(list *[]flow.Identifier) func(flow.Identifier) error { + return func(blockID flow.Identifier) error { + *list = append(*list, blockID) + return nil + } +} + +func TestNewFinalizer(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + headers := &mockstor.Headers{} + state := &mockprot.FollowerState{} + tracer := trace.NewNoopTracer() + fin := NewFinalizer(db, headers, state, tracer) + assert.Equal(t, fin.db, db) + assert.Equal(t, fin.headers, headers) + assert.Equal(t, fin.state, state) + }) +} + +// TestMakeFinalValidChain checks whether calling `MakeFinal` with the ID of a valid +// descendant block of the latest finalized header results in the finalization of the +// valid descendant and all of its parents up to the finalized header, but excluding +// the children of the valid descendant. +func TestMakeFinalValidChain(t *testing.T) { + + // create one block that we consider the last finalized + final := unittest.BlockHeaderFixture() + final.Height = uint64(rand.Uint32()) + + // generate a couple of children that are pending + parent := final + var pending []*flow.Header + total := 8 + for i := 0; i < total; i++ { + header := unittest.BlockHeaderFixture() + header.Height = parent.Height + 1 + header.ParentID = parent.ID() + pending = append(pending, header) + parent = header + } + + // create a mock protocol state to check finalize calls + state := mockprot.NewFollowerState(t) + + // make sure we get a finalize call for the blocks that we want to + cutoff := total - 3 + var lastID flow.Identifier + for i := 0; i < cutoff; i++ { + state.On("Finalize", mock.Anything, pending[i].ID()).Return(nil) + lastID = pending[i].ID() + } + + // this will hold the IDs of blocks clean up + var list []flow.Identifier + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // insert the latest finalized height + err := db.Update(operation.InsertFinalizedHeight(final.Height)) + require.NoError(t, err) + + // map the finalized height to the finalized block ID + err = db.Update(operation.IndexBlockHeight(final.Height, final.ID())) + require.NoError(t, err) + + // insert the finalized block header into the DB + err = db.Update(operation.InsertHeader(final.ID(), final)) + require.NoError(t, err) + + // insert all of the pending blocks into the DB + for _, header := range pending { + err = db.Update(operation.InsertHeader(header.ID(), header)) + require.NoError(t, err) + } + + // initialize the finalizer with the dependencies and make the call + metrics := metrics.NewNoopCollector() + fin := Finalizer{ + db: db, + headers: storage.NewHeaders(metrics, db), + state: state, + tracer: trace.NewNoopTracer(), + cleanup: LogCleanup(&list), + } + err = fin.MakeFinal(lastID) + require.NoError(t, err) + }) + + // make sure that finalize was called on protocol state for all desired blocks + state.AssertExpectations(t) + + // make sure that cleanup was called for all of them too + assert.ElementsMatch(t, list, flow.GetIDs(pending[:cutoff])) +} + +// TestMakeFinalInvalidHeight checks whether we receive an error when calling `MakeFinal` +// with a header that is at the same height as the already highest finalized header. +func TestMakeFinalInvalidHeight(t *testing.T) { + + // create one block that we consider the last finalized + final := unittest.BlockHeaderFixture() + final.Height = uint64(rand.Uint32()) + + // generate an alternative block at same height + pending := unittest.BlockHeaderFixture() + pending.Height = final.Height + + // create a mock protocol state to check finalize calls + state := mockprot.NewFollowerState(t) + + // this will hold the IDs of blocks clean up + var list []flow.Identifier + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // insert the latest finalized height + err := db.Update(operation.InsertFinalizedHeight(final.Height)) + require.NoError(t, err) + + // map the finalized height to the finalized block ID + err = db.Update(operation.IndexBlockHeight(final.Height, final.ID())) + require.NoError(t, err) + + // insert the finalized block header into the DB + err = db.Update(operation.InsertHeader(final.ID(), final)) + require.NoError(t, err) + + // insert all of the pending header into DB + err = db.Update(operation.InsertHeader(pending.ID(), pending)) + require.NoError(t, err) + + // initialize the finalizer with the dependencies and make the call + metrics := metrics.NewNoopCollector() + fin := Finalizer{ + db: db, + headers: storage.NewHeaders(metrics, db), + state: state, + tracer: trace.NewNoopTracer(), + cleanup: LogCleanup(&list), + } + err = fin.MakeFinal(pending.ID()) + require.Error(t, err) + }) + + // make sure that nothing was finalized + state.AssertExpectations(t) + + // make sure no cleanup was done + assert.Empty(t, list) +} + +// TestMakeFinalDuplicate checks whether calling `MakeFinal` with the ID of the currently +// highest finalized header is a no-op and does not result in an error. +func TestMakeFinalDuplicate(t *testing.T) { + + // create one block that we consider the last finalized + final := unittest.BlockHeaderFixture() + final.Height = uint64(rand.Uint32()) + + // create a mock protocol state to check finalize calls + state := mockprot.NewFollowerState(t) + + // this will hold the IDs of blocks clean up + var list []flow.Identifier + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // insert the latest finalized height + err := db.Update(operation.InsertFinalizedHeight(final.Height)) + require.NoError(t, err) + + // map the finalized height to the finalized block ID + err = db.Update(operation.IndexBlockHeight(final.Height, final.ID())) + require.NoError(t, err) + + // insert the finalized block header into the DB + err = db.Update(operation.InsertHeader(final.ID(), final)) + require.NoError(t, err) + + // initialize the finalizer with the dependencies and make the call + metrics := metrics.NewNoopCollector() + fin := Finalizer{ + db: db, + headers: storage.NewHeaders(metrics, db), + state: state, + tracer: trace.NewNoopTracer(), + cleanup: LogCleanup(&list), + } + err = fin.MakeFinal(final.ID()) + require.NoError(t, err) + }) + + // make sure that nothing was finalized + state.AssertExpectations(t) + + // make sure no cleanup was done + assert.Empty(t, list) +} diff --git a/state/cluster/pebble/mutator.go b/state/cluster/pebble/mutator.go new file mode 100644 index 00000000000..a4d867f4a8a --- /dev/null +++ b/state/cluster/pebble/mutator.go @@ -0,0 +1,424 @@ +package badger + +import ( + "context" + "errors" + "fmt" + "math" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state" + clusterstate "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/fork" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" +) + +type MutableState struct { + *State + tracer module.Tracer + headers storage.Headers + payloads storage.ClusterPayloads +} + +var _ clusterstate.MutableState = (*MutableState)(nil) + +func NewMutableState(state *State, tracer module.Tracer, headers storage.Headers, payloads storage.ClusterPayloads) (*MutableState, error) { + mutableState := &MutableState{ + State: state, + tracer: tracer, + headers: headers, + payloads: payloads, + } + return mutableState, nil +} + +// extendContext encapsulates all state information required in order to validate a candidate cluster block. +type extendContext struct { + candidate *cluster.Block // the proposed candidate cluster block + finalizedClusterBlock *flow.Header // the latest finalized cluster block + finalizedConsensusHeight uint64 // the latest finalized height on the main chain + epochFirstHeight uint64 // the first height of this cluster's operating epoch + epochLastHeight uint64 // the last height of this cluster's operating epoch (may be unknown) + epochHasEnded bool // whether this cluster's operating epoch has ended (whether the above field is known) +} + +// getExtendCtx reads all required information from the database in order to validate +// a candidate cluster block. +// No errors are expected during normal operation. +func (m *MutableState) getExtendCtx(candidate *cluster.Block) (extendContext, error) { + var ctx extendContext + ctx.candidate = candidate + + err := m.State.db.View(func(tx *badger.Txn) error { + // get the latest finalized cluster block and latest finalized consensus height + ctx.finalizedClusterBlock = new(flow.Header) + err := procedure.RetrieveLatestFinalizedClusterHeader(candidate.Header.ChainID, ctx.finalizedClusterBlock)(tx) + if err != nil { + return fmt.Errorf("could not retrieve finalized cluster head: %w", err) + } + err = operation.RetrieveFinalizedHeight(&ctx.finalizedConsensusHeight)(tx) + if err != nil { + return fmt.Errorf("could not retrieve finalized height on consensus chain: %w", err) + } + + err = operation.RetrieveEpochFirstHeight(m.State.epoch, &ctx.epochFirstHeight)(tx) + if err != nil { + return fmt.Errorf("could not get operating epoch first height: %w", err) + } + err = operation.RetrieveEpochLastHeight(m.State.epoch, &ctx.epochLastHeight)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + ctx.epochHasEnded = false + return nil + } + return fmt.Errorf("unexpected failure to retrieve final height of operating epoch: %w", err) + } + ctx.epochHasEnded = true + return nil + }) + if err != nil { + return extendContext{}, fmt.Errorf("could not read required state information for Extend checks: %w", err) + } + return ctx, nil +} + +// Extend introduces the given block into the cluster state as a pending +// without modifying the current finalized state. +// The block's parent must have already been successfully inserted. +// TODO(ramtin) pass context here +// Expected errors during normal operations: +// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) +// - state.UnverifiableExtensionError if the reference block is _not_ a known finalized block +// - state.InvalidExtensionError if the candidate block is invalid +func (m *MutableState) Extend(candidate *cluster.Block) error { + parentSpan, ctx := m.tracer.StartCollectionSpan(context.Background(), candidate.ID(), trace.COLClusterStateMutatorExtend) + defer parentSpan.End() + + span, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckHeader) + err := m.checkHeaderValidity(candidate) + span.End() + if err != nil { + return fmt.Errorf("error checking header validity: %w", err) + } + + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendGetExtendCtx) + extendCtx, err := m.getExtendCtx(candidate) + span.End() + if err != nil { + return fmt.Errorf("error gettting extend context data: %w", err) + } + + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckAncestry) + err = m.checkConnectsToFinalizedState(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking connection to finalized state: %w", err) + } + + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckReferenceBlock) + err = m.checkPayloadReferenceBlock(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking reference block: %w", err) + } + + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckTransactionsValid) + err = m.checkPayloadTransactions(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking payload transactions: %w", err) + } + + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendDBInsert) + err = operation.RetryOnConflict(m.State.db.Update, procedure.InsertClusterBlock(candidate)) + span.End() + if err != nil { + return fmt.Errorf("could not insert cluster block: %w", err) + } + return nil +} + +// checkHeaderValidity validates that the candidate block has a header which is +// valid generally for inclusion in the cluster consensus, and w.r.t. its parent. +// Expected error returns: +// - state.InvalidExtensionError if the candidate header is invalid +func (m *MutableState) checkHeaderValidity(candidate *cluster.Block) error { + header := candidate.Header + + // check chain ID + if header.ChainID != m.State.clusterID { + return state.NewInvalidExtensionErrorf("new block chain ID (%s) does not match configured (%s)", header.ChainID, m.State.clusterID) + } + + // get the header of the parent of the new block + parent, err := m.headers.ByBlockID(header.ParentID) + if err != nil { + return irrecoverable.NewExceptionf("could not retrieve latest finalized header: %w", err) + } + + // extending block must have correct parent view + if header.ParentView != parent.View { + return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", + header.ParentView, parent.View) + } + + // the extending block must increase height by 1 from parent + if header.Height != parent.Height+1 { + return state.NewInvalidExtensionErrorf("extending block height (%d) must be parent height + 1 (%d)", + header.Height, parent.Height) + } + return nil +} + +// checkConnectsToFinalizedState validates that the candidate block connects to +// the latest finalized state (ie. is not extending an orphaned fork). +// Expected error returns: +// - state.OutdatedExtensionError if the candidate extends an orphaned fork +func (m *MutableState) checkConnectsToFinalizedState(ctx extendContext) error { + header := ctx.candidate.Header + finalizedID := ctx.finalizedClusterBlock.ID() + finalizedHeight := ctx.finalizedClusterBlock.Height + + // start with the extending block's parent + parentID := header.ParentID + for parentID != finalizedID { + // get the parent of current block + ancestor, err := m.headers.ByBlockID(parentID) + if err != nil { + return irrecoverable.NewExceptionf("could not get parent which must be known (%x): %w", header.ParentID, err) + } + + // if its height is below current boundary, the block does not connect + // to the finalized protocol state and would break database consistency + if ancestor.Height < finalizedHeight { + return state.NewOutdatedExtensionErrorf( + "block doesn't connect to latest finalized block (height=%d, id=%x): orphaned ancestor (height=%d, id=%x)", + finalizedHeight, finalizedID, ancestor.Height, parentID) + } + parentID = ancestor.ParentID + } + return nil +} + +// checkPayloadReferenceBlock validates the reference block is valid. +// - it must be a known, finalized block on the main consensus chain +// - it must be within the cluster's operating epoch +// +// Expected error returns: +// - state.InvalidExtensionError if the reference block is invalid for use. +// - state.UnverifiableExtensionError if the reference block is unknown. +func (m *MutableState) checkPayloadReferenceBlock(ctx extendContext) error { + payload := ctx.candidate.Payload + + // 1 - the reference block must be known + refBlock, err := m.headers.ByBlockID(payload.ReferenceBlockID) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return state.NewUnverifiableExtensionError("cluster block references unknown reference block (id=%x)", payload.ReferenceBlockID) + } + return fmt.Errorf("could not check reference block: %w", err) + } + + // 2 - the reference block must be finalized + if refBlock.Height > ctx.finalizedConsensusHeight { + // a reference block which is above the finalized boundary can't be verified yet + return state.NewUnverifiableExtensionError("reference block is above finalized boundary (%d>%d)", refBlock.Height, ctx.finalizedConsensusHeight) + } else { + storedBlockIDForHeight, err := m.headers.BlockIDByHeight(refBlock.Height) + if err != nil { + return irrecoverable.NewExceptionf("could not look up block ID for finalized height: %w", err) + } + // a reference block with height at or below the finalized boundary must have been finalized + if storedBlockIDForHeight != payload.ReferenceBlockID { + return state.NewInvalidExtensionErrorf("cluster block references orphaned reference block (id=%x, height=%d), the block finalized at this height is %x", + payload.ReferenceBlockID, refBlock.Height, storedBlockIDForHeight) + } + } + + // TODO ensure the reference block is part of the main chain https://github.com/onflow/flow-go/issues/4204 + _ = refBlock + + // 3 - the reference block must be within the cluster's operating epoch + if refBlock.Height < ctx.epochFirstHeight { + return state.NewInvalidExtensionErrorf("invalid reference block is before operating epoch for cluster, height %d<%d", refBlock.Height, ctx.epochFirstHeight) + } + if ctx.epochHasEnded && refBlock.Height > ctx.epochLastHeight { + return state.NewInvalidExtensionErrorf("invalid reference block is after operating epoch for cluster, height %d>%d", refBlock.Height, ctx.epochLastHeight) + } + return nil +} + +// checkPayloadTransactions validates the transactions included int the candidate cluster block's payload. +// It enforces: +// - transactions are individually valid +// - no duplicate transaction exists along the fork being extended +// - the collection's reference block is equal to the oldest reference block among +// its constituent transactions +// +// Expected error returns: +// - state.InvalidExtensionError if the reference block is invalid for use. +// - state.UnverifiableExtensionError if the reference block is unknown. +func (m *MutableState) checkPayloadTransactions(ctx extendContext) error { + block := ctx.candidate + payload := block.Payload + + if payload.Collection.Len() == 0 { + return nil + } + + // check that all transactions within the collection are valid + // keep track of the min/max reference blocks - the collection must be non-empty + // at this point so these are guaranteed to be set correctly + minRefID := flow.ZeroID + minRefHeight := uint64(math.MaxUint64) + maxRefHeight := uint64(0) + for _, flowTx := range payload.Collection.Transactions { + refBlock, err := m.headers.ByBlockID(flowTx.ReferenceBlockID) + if errors.Is(err, storage.ErrNotFound) { + // unknown reference blocks are invalid + return state.NewUnverifiableExtensionError("collection contains tx (tx_id=%x) with unknown reference block (block_id=%x): %w", flowTx.ID(), flowTx.ReferenceBlockID, err) + } + if err != nil { + return fmt.Errorf("could not check reference block (id=%x): %w", flowTx.ReferenceBlockID, err) + } + + if refBlock.Height < minRefHeight { + minRefHeight = refBlock.Height + minRefID = flowTx.ReferenceBlockID + } + if refBlock.Height > maxRefHeight { + maxRefHeight = refBlock.Height + } + } + + // a valid collection must reference the oldest reference block among + // its constituent transactions + if minRefID != payload.ReferenceBlockID { + return state.NewInvalidExtensionErrorf( + "reference block (id=%x) must match oldest transaction's reference block (id=%x)", + payload.ReferenceBlockID, minRefID, + ) + } + // a valid collection must contain only transactions within its expiry window + if maxRefHeight-minRefHeight >= flow.DefaultTransactionExpiry { + return state.NewInvalidExtensionErrorf( + "collection contains reference height range [%d,%d] exceeding expiry window size: %d", + minRefHeight, maxRefHeight, flow.DefaultTransactionExpiry) + } + + // check for duplicate transactions in block's ancestry + txLookup := make(map[flow.Identifier]struct{}) + for _, tx := range block.Payload.Collection.Transactions { + txID := tx.ID() + if _, exists := txLookup[txID]; exists { + return state.NewInvalidExtensionErrorf("collection contains transaction (id=%x) more than once", txID) + } + txLookup[txID] = struct{}{} + } + + // first, check for duplicate transactions in the un-finalized ancestry + duplicateTxIDs, err := m.checkDupeTransactionsInUnfinalizedAncestry(block, txLookup, ctx.finalizedClusterBlock.Height) + if err != nil { + return fmt.Errorf("could not check for duplicate txs in un-finalized ancestry: %w", err) + } + if len(duplicateTxIDs) > 0 { + return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in un-finalized ancestry (duplicates: %s)", duplicateTxIDs) + } + + // second, check for duplicate transactions in the finalized ancestry + duplicateTxIDs, err = m.checkDupeTransactionsInFinalizedAncestry(txLookup, minRefHeight, maxRefHeight) + if err != nil { + return fmt.Errorf("could not check for duplicate txs in finalized ancestry: %w", err) + } + if len(duplicateTxIDs) > 0 { + return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in finalized ancestry (duplicates: %s)", duplicateTxIDs) + } + + return nil +} + +// checkDupeTransactionsInUnfinalizedAncestry checks for duplicate transactions in the un-finalized +// ancestry of the given block, and returns a list of all duplicates if there are any. +func (m *MutableState) checkDupeTransactionsInUnfinalizedAncestry(block *cluster.Block, includedTransactions map[flow.Identifier]struct{}, finalHeight uint64) ([]flow.Identifier, error) { + + var duplicateTxIDs []flow.Identifier + err := fork.TraverseBackward(m.headers, block.Header.ParentID, func(ancestor *flow.Header) error { + payload, err := m.payloads.ByBlockID(ancestor.ID()) + if err != nil { + return fmt.Errorf("could not retrieve ancestor payload: %w", err) + } + + for _, tx := range payload.Collection.Transactions { + txID := tx.ID() + _, duplicated := includedTransactions[txID] + if duplicated { + duplicateTxIDs = append(duplicateTxIDs, txID) + } + } + return nil + }, fork.ExcludingHeight(finalHeight)) + + return duplicateTxIDs, err +} + +// checkDupeTransactionsInFinalizedAncestry checks for duplicate transactions in the finalized +// ancestry, and returns a list of all duplicates if there are any. +func (m *MutableState) checkDupeTransactionsInFinalizedAncestry(includedTransactions map[flow.Identifier]struct{}, minRefHeight, maxRefHeight uint64) ([]flow.Identifier, error) { + var duplicatedTxIDs []flow.Identifier + + // Let E be the global transaction expiry constant, measured in blocks. For each + // T ∈ `includedTransactions`, we have to decide whether the transaction + // already appeared in _any_ finalized cluster block. + // Notation: + // - consider a valid cluster block C and let c be its reference block height + // - consider a transaction T ∈ `includedTransactions` and let t denote its + // reference block height + // + // Boundary conditions: + // 1. C's reference block height is equal to the lowest reference block height of + // all its constituent transactions. Hence, for collection C to potentially contain T, it must satisfy c <= t. + // 2. For T to be eligible for inclusion in collection C, _none_ of the transactions within C are allowed + // to be expired w.r.t. C's reference block. Hence, for collection C to potentially contain T, it must satisfy t < c + E. + // + // Therefore, for collection C to potentially contain transaction T, it must satisfy t - E < c <= t. + // In other words, we only need to inspect collections with reference block height c ∈ (t-E, t]. + // Consequently, for a set of transactions, with `minRefHeight` (`maxRefHeight`) being the smallest (largest) + // reference block height, we only need to inspect collections with c ∈ (minRefHeight-E, maxRefHeight]. + + // the finalized cluster blocks which could possibly contain any conflicting transactions + var clusterBlockIDs []flow.Identifier + start := minRefHeight - flow.DefaultTransactionExpiry + 1 + if start > minRefHeight { + start = 0 // overflow check + } + end := maxRefHeight + err := m.db.View(operation.LookupClusterBlocksByReferenceHeightRange(start, end, &clusterBlockIDs)) + if err != nil { + return nil, fmt.Errorf("could not lookup finalized cluster blocks by reference height range [%d,%d]: %w", start, end, err) + } + + for _, blockID := range clusterBlockIDs { + // TODO: could add LightByBlockID and retrieve only tx IDs + payload, err := m.payloads.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("could not retrieve cluster payload (block_id=%x) to de-duplicate: %w", blockID, err) + } + for _, tx := range payload.Collection.Transactions { + txID := tx.ID() + _, duplicated := includedTransactions[txID] + if duplicated { + duplicatedTxIDs = append(duplicatedTxIDs, txID) + } + } + } + + return duplicatedTxIDs, nil +} diff --git a/state/cluster/pebble/mutator_test.go b/state/cluster/pebble/mutator_test.go new file mode 100644 index 00000000000..1897cf6a39a --- /dev/null +++ b/state/cluster/pebble/mutator_test.go @@ -0,0 +1,616 @@ +package badger + +import ( + "context" + "fmt" + "math" + "math/rand" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + model "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state" + "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" + pbadger "github.com/onflow/flow-go/state/protocol/badger" + "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/state/protocol/inmem" + protocolutil "github.com/onflow/flow-go/state/protocol/util" + storage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/util" + "github.com/onflow/flow-go/utils/unittest" +) + +type MutatorSuite struct { + suite.Suite + db *badger.DB + dbdir string + + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 + + // protocol state for reference blocks for transactions + protoState protocol.FollowerState + protoGenesis *flow.Header + + state cluster.MutableState +} + +// runs before each test runs +func (suite *MutatorSuite) SetupTest() { + var err error + + suite.genesis = model.Genesis() + suite.chainID = suite.genesis.Header.ChainID + + suite.dbdir = unittest.TempDir(suite.T()) + suite.db = unittest.BadgerDB(suite.T(), suite.dbdir) + + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := util.StorageLayer(suite.T(), suite.db) + colPayloads := storage.NewClusterPayloads(metrics, suite.db) + + // just bootstrap with a genesis block, we'll use this as reference + genesis, result, seal := unittest.BootstrapFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + // ensure we don't enter a new epoch for tests that build many blocks + result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = genesis.Header.View + 100_000 + seal.ResultID = result.ID() + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) + rootSnapshot, err := inmem.SnapshotFromBootstrapState(genesis, result, seal, qc) + require.NoError(suite.T(), err) + suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter + + suite.protoGenesis = genesis.Header + state, err := pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(suite.T(), err) + suite.protoState, err = pbadger.NewFollowerState(log, tracer, events.NewNoop(), state, all.Index, all.Payloads, protocolutil.MockBlockTimer()) + require.NoError(suite.T(), err) + + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.NoError(err) + clusterState, err := Bootstrap(suite.db, clusterStateRoot) + suite.Assert().Nil(err) + suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) + suite.Assert().Nil(err) +} + +// runs after each test finishes +func (suite *MutatorSuite) TearDownTest() { + err := suite.db.Close() + suite.Assert().Nil(err) + err = os.RemoveAll(suite.dbdir) + suite.Assert().Nil(err) +} + +// Payload returns a valid cluster block payload containing the given transactions. +func (suite *MutatorSuite) Payload(transactions ...*flow.TransactionBody) model.Payload { + final, err := suite.protoState.Final().Head() + suite.Require().Nil(err) + + // find the oldest reference block among the transactions + minRefID := final.ID() // use final by default + minRefHeight := uint64(math.MaxUint64) + for _, tx := range transactions { + refBlock, err := suite.protoState.AtBlockID(tx.ReferenceBlockID).Head() + if err != nil { + continue + } + if refBlock.Height < minRefHeight { + minRefHeight = refBlock.Height + minRefID = refBlock.ID() + } + } + return model.PayloadFromTransactions(minRefID, transactions...) +} + +// BlockWithParent returns a valid block with the given parent. +func (suite *MutatorSuite) BlockWithParent(parent *model.Block) model.Block { + block := unittest.ClusterBlockWithParent(parent) + payload := suite.Payload() + block.SetPayload(payload) + return block +} + +// Block returns a valid cluster block with genesis as parent. +func (suite *MutatorSuite) Block() model.Block { + return suite.BlockWithParent(suite.genesis) +} + +func (suite *MutatorSuite) FinalizeBlock(block model.Block) { + err := suite.db.Update(func(tx *badger.Txn) error { + var refBlock flow.Header + err := operation.RetrieveHeader(block.Payload.ReferenceBlockID, &refBlock)(tx) + if err != nil { + return err + } + err = procedure.FinalizeClusterBlock(block.ID())(tx) + if err != nil { + return err + } + err = operation.IndexClusterBlockByReferenceHeight(refBlock.Height, block.ID())(tx) + return err + }) + suite.Assert().NoError(err) +} + +func (suite *MutatorSuite) Tx(opts ...func(*flow.TransactionBody)) flow.TransactionBody { + final, err := suite.protoState.Final().Head() + suite.Require().Nil(err) + + tx := unittest.TransactionBodyFixture(opts...) + tx.ReferenceBlockID = final.ID() + return tx +} + +func TestMutator(t *testing.T) { + suite.Run(t, new(MutatorSuite)) +} + +func (suite *MutatorSuite) TestBootstrap_InvalidHeight() { + suite.genesis.Header.Height = 1 + + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Assert().Error(err) +} + +func (suite *MutatorSuite) TestBootstrap_InvalidParentHash() { + suite.genesis.Header.ParentID = unittest.IdentifierFixture() + + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Assert().Error(err) +} + +func (suite *MutatorSuite) TestBootstrap_InvalidPayloadHash() { + suite.genesis.Header.PayloadHash = unittest.IdentifierFixture() + + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Assert().Error(err) +} + +func (suite *MutatorSuite) TestBootstrap_InvalidPayload() { + // this is invalid because genesis collection should be empty + suite.genesis.Payload = unittest.ClusterPayloadFixture(2) + + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Assert().Error(err) +} + +func (suite *MutatorSuite) TestBootstrap_Successful() { + err := suite.db.View(func(tx *badger.Txn) error { + + // should insert collection + var collection flow.LightCollection + err := operation.RetrieveCollection(suite.genesis.Payload.Collection.ID(), &collection)(tx) + suite.Assert().Nil(err) + suite.Assert().Equal(suite.genesis.Payload.Collection.Light(), collection) + + // should index collection + collection = flow.LightCollection{} // reset the collection + err = operation.LookupCollectionPayload(suite.genesis.ID(), &collection.Transactions)(tx) + suite.Assert().Nil(err) + suite.Assert().Equal(suite.genesis.Payload.Collection.Light(), collection) + + // should insert header + var header flow.Header + err = operation.RetrieveHeader(suite.genesis.ID(), &header)(tx) + suite.Assert().Nil(err) + suite.Assert().Equal(suite.genesis.Header.ID(), header.ID()) + + // should insert block height -> ID lookup + var blockID flow.Identifier + err = operation.LookupClusterBlockHeight(suite.genesis.Header.ChainID, suite.genesis.Header.Height, &blockID)(tx) + suite.Assert().Nil(err) + suite.Assert().Equal(suite.genesis.ID(), blockID) + + // should insert boundary + var boundary uint64 + err = operation.RetrieveClusterFinalizedHeight(suite.genesis.Header.ChainID, &boundary)(tx) + suite.Assert().Nil(err) + suite.Assert().Equal(suite.genesis.Header.Height, boundary) + + return nil + }) + suite.Assert().Nil(err) +} + +func (suite *MutatorSuite) TestExtend_WithoutBootstrap() { + block := unittest.ClusterBlockWithParent(suite.genesis) + err := suite.state.Extend(&block) + suite.Assert().Error(err) +} + +func (suite *MutatorSuite) TestExtend_InvalidChainID() { + block := suite.Block() + // change the chain ID + block.Header.ChainID = flow.ChainID(fmt.Sprintf("%s-invalid", block.Header.ChainID)) + + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_InvalidBlockHeight() { + block := suite.Block() + // change the block height + block.Header.Height = block.Header.Height - 1 + + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +// TestExtend_InvalidParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent +// with view of block referred by ParentID. +func (suite *MutatorSuite) TestExtend_InvalidParentView() { + block := suite.Block() + // change the block parent view + block.Header.ParentView-- + + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_DuplicateTxInPayload() { + block := suite.Block() + // add the same transaction to a payload twice + tx := suite.Tx() + payload := suite.Payload(&tx, &tx) + block.SetPayload(payload) + + // should fail to extend block with invalid payload + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_OnParentOfFinalized() { + // build one block on top of genesis + block1 := suite.Block() + err := suite.state.Extend(&block1) + suite.Assert().Nil(err) + + // finalize the block + suite.FinalizeBlock(block1) + + // insert another block on top of genesis + // since we have already finalized block 1, this is invalid + block2 := suite.Block() + + // try to extend with the invalid block + err = suite.state.Extend(&block2) + suite.Assert().Error(err) + suite.Assert().True(state.IsOutdatedExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_Success() { + block := suite.Block() + err := suite.state.Extend(&block) + suite.Assert().Nil(err) + + // should be able to retrieve the block + var extended model.Block + err = suite.db.View(procedure.RetrieveClusterBlock(block.ID(), &extended)) + suite.Assert().Nil(err) + suite.Assert().Equal(*block.Payload, *extended.Payload) + + // the block should be indexed by its parent + var childIDs flow.IdentifierList + err = suite.db.View(procedure.LookupBlockChildren(suite.genesis.ID(), &childIDs)) + suite.Assert().Nil(err) + suite.Require().Len(childIDs, 1) + suite.Assert().Equal(block.ID(), childIDs[0]) +} + +func (suite *MutatorSuite) TestExtend_WithEmptyCollection() { + block := suite.Block() + // set an empty collection as the payload + block.SetPayload(suite.Payload()) + err := suite.state.Extend(&block) + suite.Assert().Nil(err) +} + +// an unknown reference block is unverifiable +func (suite *MutatorSuite) TestExtend_WithNonExistentReferenceBlock() { + suite.Run("empty collection", func() { + block := suite.Block() + block.Payload.ReferenceBlockID = unittest.IdentifierFixture() + block.SetPayload(*block.Payload) + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) + }) + suite.Run("non-empty collection", func() { + block := suite.Block() + tx := suite.Tx() + payload := suite.Payload(&tx) + // set a random reference block ID + payload.ReferenceBlockID = unittest.IdentifierFixture() + block.SetPayload(payload) + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) + }) +} + +// a collection with an expired reference block is a VALID extension of chain state +func (suite *MutatorSuite) TestExtend_WithExpiredReferenceBlock() { + // build enough blocks so that using genesis as a reference block causes + // the collection to be expired + parent := suite.protoGenesis + for i := 0; i < flow.DefaultTransactionExpiry+1; i++ { + next := unittest.BlockWithParentFixture(parent) + next.Payload.Guarantees = nil + next.SetPayload(*next.Payload) + err := suite.protoState.ExtendCertified(context.Background(), next, unittest.CertifyBlock(next.Header)) + suite.Require().Nil(err) + err = suite.protoState.Finalize(context.Background(), next.ID()) + suite.Require().Nil(err) + parent = next.Header + } + + block := suite.Block() + // set genesis as reference block + block.SetPayload(model.EmptyPayload(suite.protoGenesis.ID())) + err := suite.state.Extend(&block) + suite.Assert().Nil(err) +} + +func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromClusterChain() { + // TODO skipping as this isn't implemented yet + unittest.SkipUnless(suite.T(), unittest.TEST_TODO, "skipping as this isn't implemented yet") + + block := suite.Block() + // set genesis from cluster chain as reference block + block.SetPayload(model.EmptyPayload(suite.genesis.ID())) + err := suite.state.Extend(&block) + suite.Assert().Error(err) +} + +// TestExtend_WithReferenceBlockFromDifferentEpoch tests extending the cluster state +// using a reference block in a different epoch than the cluster's epoch. +func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromDifferentEpoch() { + // build and complete the current epoch, then use a reference block from next epoch + eb := unittest.NewEpochBuilder(suite.T(), suite.protoState) + eb.BuildEpoch().CompleteEpoch() + heights, ok := eb.EpochHeights(1) + require.True(suite.T(), ok) + nextEpochHeader, err := suite.protoState.AtHeight(heights.FinalHeight() + 1).Head() + require.NoError(suite.T(), err) + + block := suite.Block() + block.SetPayload(model.EmptyPayload(nextEpochHeader.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +// TestExtend_WithUnfinalizedReferenceBlock tests that extending the cluster state +// with a reference block which is un-finalized and above the finalized boundary +// should be considered an unverifiable extension. It's possible that this reference +// block has been finalized, we just haven't processed it yet. +func (suite *MutatorSuite) TestExtend_WithUnfinalizedReferenceBlock() { + unfinalized := unittest.BlockWithParentFixture(suite.protoGenesis) + unfinalized.Payload.Guarantees = nil + unfinalized.SetPayload(*unfinalized.Payload) + err := suite.protoState.ExtendCertified(context.Background(), unfinalized, unittest.CertifyBlock(unfinalized.Header)) + suite.Require().NoError(err) + + block := suite.Block() + block.SetPayload(model.EmptyPayload(unfinalized.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) +} + +// TestExtend_WithOrphanedReferenceBlock tests that extending the cluster state +// with a un-finalized reference block below the finalized boundary +// (i.e. orphaned) should be considered an invalid extension. As the proposer is supposed +// to only use finalized blocks as reference, the proposer knowingly generated an invalid +func (suite *MutatorSuite) TestExtend_WithOrphanedReferenceBlock() { + // create a block extending genesis which is not finalized + orphaned := unittest.BlockWithParentFixture(suite.protoGenesis) + err := suite.protoState.ExtendCertified(context.Background(), orphaned, unittest.CertifyBlock(orphaned.Header)) + suite.Require().NoError(err) + + // create a block extending genesis (conflicting with previous) which is finalized + finalized := unittest.BlockWithParentFixture(suite.protoGenesis) + finalized.Payload.Guarantees = nil + finalized.SetPayload(*finalized.Payload) + err = suite.protoState.ExtendCertified(context.Background(), finalized, unittest.CertifyBlock(finalized.Header)) + suite.Require().NoError(err) + err = suite.protoState.Finalize(context.Background(), finalized.ID()) + suite.Require().NoError(err) + + // test referencing the orphaned block + block := suite.Block() + block.SetPayload(model.EmptyPayload(orphaned.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_UnfinalizedBlockWithDupeTx() { + tx1 := suite.Tx() + + // create a block extending genesis containing tx1 + block1 := suite.Block() + payload1 := suite.Payload(&tx1) + block1.SetPayload(payload1) + + // should be able to extend block 1 + err := suite.state.Extend(&block1) + suite.Assert().Nil(err) + + // create a block building on block1 ALSO containing tx1 + block2 := suite.BlockWithParent(&block1) + payload2 := suite.Payload(&tx1) + block2.SetPayload(payload2) + + // should be unable to extend block 2, as it contains a dupe transaction + err = suite.state.Extend(&block2) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_FinalizedBlockWithDupeTx() { + tx1 := suite.Tx() + + // create a block extending genesis containing tx1 + block1 := suite.Block() + payload1 := suite.Payload(&tx1) + block1.SetPayload(payload1) + + // should be able to extend block 1 + err := suite.state.Extend(&block1) + suite.Assert().Nil(err) + + // should be able to finalize block 1 + suite.FinalizeBlock(block1) + suite.Assert().Nil(err) + + // create a block building on block1 ALSO containing tx1 + block2 := suite.BlockWithParent(&block1) + payload2 := suite.Payload(&tx1) + block2.SetPayload(payload2) + + // should be unable to extend block 2, as it contains a dupe transaction + err = suite.state.Extend(&block2) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +func (suite *MutatorSuite) TestExtend_ConflictingForkWithDupeTx() { + tx1 := suite.Tx() + + // create a block extending genesis containing tx1 + block1 := suite.Block() + payload1 := suite.Payload(&tx1) + block1.SetPayload(payload1) + + // should be able to extend block 1 + err := suite.state.Extend(&block1) + suite.Assert().Nil(err) + + // create a block ALSO extending genesis ALSO containing tx1 + block2 := suite.Block() + payload2 := suite.Payload(&tx1) + block2.SetPayload(payload2) + + // should be able to extend block2 + // although it conflicts with block1, it is on a different fork + err = suite.state.Extend(&block2) + suite.Assert().Nil(err) +} + +func (suite *MutatorSuite) TestExtend_LargeHistory() { + t := suite.T() + + // get a valid reference block ID + final, err := suite.protoState.Final().Head() + require.NoError(t, err) + refID := final.ID() + + // keep track of the head of the chain + head := *suite.genesis + + // keep track of transactions in orphaned forks (eligible for inclusion in future block) + var invalidatedTransactions []*flow.TransactionBody + // keep track of the oldest transactions (further back in ancestry than the expiry window) + var oldTransactions []*flow.TransactionBody + + // create a large history of blocks with invalidated forks every 3 blocks on + // average - build until the height exceeds transaction expiry + for i := 0; ; i++ { + + // create a transaction + tx := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { + tx.ReferenceBlockID = refID + tx.ProposalKey.SequenceNumber = uint64(i) + }) + + // 1/3 of the time create a conflicting fork that will be invalidated + // don't do this the first and last few times to ensure we don't + // try to fork genesis and the last block is the valid fork. + conflicting := rand.Intn(3) == 0 && i > 5 && i < 995 + + // by default, build on the head - if we are building a + // conflicting fork, build on the parent of the head + parent := head + if conflicting { + err = suite.db.View(procedure.RetrieveClusterBlock(parent.Header.ParentID, &parent)) + assert.NoError(t, err) + // add the transaction to the invalidated list + invalidatedTransactions = append(invalidatedTransactions, &tx) + } else if head.Header.Height < 50 { + oldTransactions = append(oldTransactions, &tx) + } + + // create a block containing the transaction + block := unittest.ClusterBlockWithParent(&head) + payload := suite.Payload(&tx) + block.SetPayload(payload) + err = suite.state.Extend(&block) + assert.NoError(t, err) + + // reset the valid head if we aren't building a conflicting fork + if !conflicting { + head = block + suite.FinalizeBlock(block) + assert.NoError(t, err) + } + + // stop building blocks once we've built a history which exceeds the transaction + // expiry length - this tests that deduplication works properly against old blocks + // which nevertheless have a potentially conflicting reference block + if head.Header.Height > flow.DefaultTransactionExpiry+100 { + break + } + } + + t.Log("conflicting: ", len(invalidatedTransactions)) + + t.Run("should be able to extend with transactions in orphaned forks", func(t *testing.T) { + block := unittest.ClusterBlockWithParent(&head) + payload := suite.Payload(invalidatedTransactions...) + block.SetPayload(payload) + err = suite.state.Extend(&block) + assert.NoError(t, err) + }) + + t.Run("should be unable to extend with conflicting transactions within reference height range of extending block", func(t *testing.T) { + block := unittest.ClusterBlockWithParent(&head) + payload := suite.Payload(oldTransactions...) + block.SetPayload(payload) + err = suite.state.Extend(&block) + assert.Error(t, err) + suite.Assert().True(state.IsInvalidExtensionError(err)) + }) +} diff --git a/state/cluster/pebble/params.go b/state/cluster/pebble/params.go new file mode 100644 index 00000000000..ab557f2a7f2 --- /dev/null +++ b/state/cluster/pebble/params.go @@ -0,0 +1,13 @@ +package badger + +import ( + "github.com/onflow/flow-go/model/flow" +) + +type Params struct { + state *State +} + +func (p *Params) ChainID() (flow.ChainID, error) { + return p.state.clusterID, nil +} diff --git a/state/cluster/pebble/snapshot.go b/state/cluster/pebble/snapshot.go new file mode 100644 index 00000000000..7823f700163 --- /dev/null +++ b/state/cluster/pebble/snapshot.go @@ -0,0 +1,102 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" +) + +// Snapshot represents a snapshot of chain state anchored at a particular +// reference block. +type Snapshot struct { + err error + state *State + blockID flow.Identifier +} + +func (s *Snapshot) Collection() (*flow.Collection, error) { + if s.err != nil { + return nil, s.err + } + + var collection flow.Collection + err := s.state.db.View(func(tx *badger.Txn) error { + + // get the header for this snapshot + var header flow.Header + err := s.head(&header)(tx) + if err != nil { + return fmt.Errorf("failed to get snapshot header: %w", err) + } + + // get the payload + var payload cluster.Payload + err = procedure.RetrieveClusterPayload(header.ID(), &payload)(tx) + if err != nil { + return fmt.Errorf("failed to get snapshot payload: %w", err) + } + + // set the collection + collection = payload.Collection + + return nil + }) + + return &collection, err +} + +func (s *Snapshot) Head() (*flow.Header, error) { + if s.err != nil { + return nil, s.err + } + + var head flow.Header + err := s.state.db.View(func(tx *badger.Txn) error { + return s.head(&head)(tx) + }) + return &head, err +} + +func (s *Snapshot) Pending() ([]flow.Identifier, error) { + if s.err != nil { + return nil, s.err + } + return s.pending(s.blockID) +} + +// head finds the header referenced by the snapshot. +func (s *Snapshot) head(head *flow.Header) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // get the snapshot header + err := operation.RetrieveHeader(s.blockID, head)(tx) + if err != nil { + return fmt.Errorf("could not retrieve header for block (%s): %w", s.blockID, err) + } + + return nil + } +} + +func (s *Snapshot) pending(blockID flow.Identifier) ([]flow.Identifier, error) { + + var pendingIDs flow.IdentifierList + err := s.state.db.View(procedure.LookupBlockChildren(blockID, &pendingIDs)) + if err != nil { + return nil, fmt.Errorf("could not get pending children: %w", err) + } + + for _, pendingID := range pendingIDs { + additionalIDs, err := s.pending(pendingID) + if err != nil { + return nil, fmt.Errorf("could not get pending grandchildren: %w", err) + } + pendingIDs = append(pendingIDs, additionalIDs...) + } + return pendingIDs, nil +} diff --git a/state/cluster/pebble/snapshot_test.go b/state/cluster/pebble/snapshot_test.go new file mode 100644 index 00000000000..7dd81c0ed4d --- /dev/null +++ b/state/cluster/pebble/snapshot_test.go @@ -0,0 +1,297 @@ +package badger + +import ( + "math" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + model "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" + pbadger "github.com/onflow/flow-go/state/protocol/badger" + storage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/util" + "github.com/onflow/flow-go/utils/unittest" +) + +type SnapshotSuite struct { + suite.Suite + db *badger.DB + dbdir string + + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 + + protoState protocol.State + + state cluster.MutableState +} + +// runs before each test runs +func (suite *SnapshotSuite) SetupTest() { + var err error + + suite.genesis = model.Genesis() + suite.chainID = suite.genesis.Header.ChainID + + suite.dbdir = unittest.TempDir(suite.T()) + suite.db = unittest.BadgerDB(suite.T(), suite.dbdir) + + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + + all := util.StorageLayer(suite.T(), suite.db) + colPayloads := storage.NewClusterPayloads(metrics, suite.db) + + root := unittest.RootSnapshotFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + suite.epochCounter = root.Encodable().Epochs.Current.Counter + + suite.protoState, err = pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + root, + ) + suite.Require().NoError(err) + + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Require().NoError(err) + clusterState, err := Bootstrap(suite.db, clusterStateRoot) + suite.Require().NoError(err) + suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) + suite.Require().NoError(err) +} + +// runs after each test finishes +func (suite *SnapshotSuite) TearDownTest() { + err := suite.db.Close() + suite.Assert().Nil(err) + err = os.RemoveAll(suite.dbdir) + suite.Assert().Nil(err) +} + +// Payload returns a valid cluster block payload containing the given transactions. +func (suite *SnapshotSuite) Payload(transactions ...*flow.TransactionBody) model.Payload { + final, err := suite.protoState.Final().Head() + suite.Require().Nil(err) + + // find the oldest reference block among the transactions + minRefID := final.ID() // use final by default + minRefHeight := uint64(math.MaxUint64) + for _, tx := range transactions { + refBlock, err := suite.protoState.AtBlockID(tx.ReferenceBlockID).Head() + if err != nil { + continue + } + if refBlock.Height < minRefHeight { + minRefHeight = refBlock.Height + minRefID = refBlock.ID() + } + } + return model.PayloadFromTransactions(minRefID, transactions...) +} + +// BlockWithParent returns a valid block with the given parent. +func (suite *SnapshotSuite) BlockWithParent(parent *model.Block) model.Block { + block := unittest.ClusterBlockWithParent(parent) + payload := suite.Payload() + block.SetPayload(payload) + return block +} + +// Block returns a valid cluster block with genesis as parent. +func (suite *SnapshotSuite) Block() model.Block { + return suite.BlockWithParent(suite.genesis) +} + +func (suite *SnapshotSuite) InsertBlock(block model.Block) { + err := suite.db.Update(procedure.InsertClusterBlock(&block)) + suite.Assert().Nil(err) +} + +// InsertSubtree recursively inserts chain state as a subtree of the parent +// block. The subtree has the given depth and `fanout` children at each node. +// All child indices are updated. +func (suite *SnapshotSuite) InsertSubtree(parent model.Block, depth, fanout int) { + if depth == 0 { + return + } + + for i := 0; i < fanout; i++ { + block := suite.BlockWithParent(&parent) + suite.InsertBlock(block) + suite.InsertSubtree(block, depth-1, fanout) + } +} + +func TestSnapshot(t *testing.T) { + suite.Run(t, new(SnapshotSuite)) +} + +func (suite *SnapshotSuite) TestNonexistentBlock() { + t := suite.T() + + nonexistentBlockID := unittest.IdentifierFixture() + snapshot := suite.state.AtBlockID(nonexistentBlockID) + + _, err := snapshot.Collection() + assert.Error(t, err) + + _, err = snapshot.Head() + assert.Error(t, err) +} + +func (suite *SnapshotSuite) TestAtBlockID() { + t := suite.T() + + snapshot := suite.state.AtBlockID(suite.genesis.ID()) + + // ensure collection is correct + coll, err := snapshot.Collection() + assert.Nil(t, err) + assert.Equal(t, &suite.genesis.Payload.Collection, coll) + + // ensure head is correct + head, err := snapshot.Head() + assert.Nil(t, err) + assert.Equal(t, suite.genesis.ID(), head.ID()) +} + +func (suite *SnapshotSuite) TestEmptyCollection() { + t := suite.T() + + // create a block with an empty collection + block := suite.BlockWithParent(suite.genesis) + block.SetPayload(model.EmptyPayload(flow.ZeroID)) + suite.InsertBlock(block) + + snapshot := suite.state.AtBlockID(block.ID()) + + // ensure collection is correct + coll, err := snapshot.Collection() + assert.Nil(t, err) + assert.Equal(t, &block.Payload.Collection, coll) +} + +func (suite *SnapshotSuite) TestFinalizedBlock() { + t := suite.T() + + // create a new finalized block on genesis (height=1) + finalizedBlock1 := suite.Block() + err := suite.state.Extend(&finalizedBlock1) + assert.Nil(t, err) + + // create an un-finalized block on genesis (height=1) + unFinalizedBlock1 := suite.Block() + err = suite.state.Extend(&unFinalizedBlock1) + assert.Nil(t, err) + + // create a second un-finalized on top of the finalized block (height=2) + unFinalizedBlock2 := suite.BlockWithParent(&finalizedBlock1) + err = suite.state.Extend(&unFinalizedBlock2) + assert.Nil(t, err) + + // finalize the block + err = suite.db.Update(procedure.FinalizeClusterBlock(finalizedBlock1.ID())) + assert.Nil(t, err) + + // get the final snapshot, should map to finalizedBlock1 + snapshot := suite.state.Final() + + // ensure collection is correct + coll, err := snapshot.Collection() + assert.Nil(t, err) + assert.Equal(t, &finalizedBlock1.Payload.Collection, coll) + + // ensure head is correct + head, err := snapshot.Head() + assert.Nil(t, err) + assert.Equal(t, finalizedBlock1.ID(), head.ID()) +} + +// test that no pending blocks are returned when there are none +func (suite *SnapshotSuite) TestPending_NoPendingBlocks() { + + // first, check that a freshly bootstrapped state has no pending blocks + suite.Run("freshly bootstrapped state", func() { + pending, err := suite.state.Final().Pending() + suite.Require().Nil(err) + suite.Assert().Len(pending, 0) + }) + +} + +// test that the appropriate pending blocks are included +func (suite *SnapshotSuite) TestPending_WithPendingBlocks() { + + // check with some finalized blocks + parent := suite.genesis + pendings := make([]flow.Identifier, 0, 10) + for i := 0; i < 10; i++ { + next := suite.BlockWithParent(parent) + suite.InsertBlock(next) + pendings = append(pendings, next.ID()) + } + + pending, err := suite.state.Final().Pending() + suite.Require().Nil(err) + suite.Require().Equal(pendings, pending) +} + +// ensure that pending blocks are included, even they aren't direct children +// of the finalized head +func (suite *SnapshotSuite) TestPending_Grandchildren() { + + // create 3 levels of children + suite.InsertSubtree(*suite.genesis, 3, 3) + + pending, err := suite.state.Final().Pending() + suite.Require().Nil(err) + + // we should have 3 + 3^2 + 3^3 = 39 total children + suite.Assert().Len(pending, 39) + + // the result must be ordered so that we see parents before their children + parents := make(map[flow.Identifier]struct{}) + // initialize with the latest finalized block, which is the parent of the + // first level of children + parents[suite.genesis.ID()] = struct{}{} + + for _, blockID := range pending { + var header flow.Header + err := suite.db.View(operation.RetrieveHeader(blockID, &header)) + suite.Require().Nil(err) + + // we must have already seen the parent + _, seen := parents[header.ParentID] + suite.Assert().True(seen, "pending list contained child (%x) before parent (%x)", header.ID(), header.ParentID) + + // mark this block as seen + parents[header.ID()] = struct{}{} + } +} + +func (suite *SnapshotSuite) TestParams_ChainID() { + + chainID, err := suite.state.Params().ChainID() + suite.Require().Nil(err) + suite.Assert().Equal(suite.genesis.Header.ChainID, chainID) +} diff --git a/state/cluster/pebble/state.go b/state/cluster/pebble/state.go new file mode 100644 index 00000000000..f088328823e --- /dev/null +++ b/state/cluster/pebble/state.go @@ -0,0 +1,165 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" +) + +type State struct { + db *badger.DB + clusterID flow.ChainID // the chain ID for the cluster + epoch uint64 // the operating epoch for the cluster +} + +// Bootstrap initializes the persistent cluster state with a genesis block. +// The genesis block must have height 0, a parent hash of 32 zero bytes, +// and an empty collection as payload. +func Bootstrap(db *badger.DB, stateRoot *StateRoot) (*State, error) { + isBootstrapped, err := IsBootstrapped(db, stateRoot.ClusterID()) + if err != nil { + return nil, fmt.Errorf("failed to determine whether database contains bootstrapped state: %w", err) + } + if isBootstrapped { + return nil, fmt.Errorf("expected empty cluster state for cluster ID %s", stateRoot.ClusterID()) + } + state := newState(db, stateRoot.ClusterID(), stateRoot.EpochCounter()) + + genesis := stateRoot.Block() + rootQC := stateRoot.QC() + // bootstrap cluster state + err = operation.RetryOnConflict(state.db.Update, func(tx *badger.Txn) error { + chainID := genesis.Header.ChainID + // insert the block + err := procedure.InsertClusterBlock(genesis)(tx) + if err != nil { + return fmt.Errorf("could not insert genesis block: %w", err) + } + // insert block height -> ID mapping + err = operation.IndexClusterBlockHeight(chainID, genesis.Header.Height, genesis.ID())(tx) + if err != nil { + return fmt.Errorf("failed to map genesis block height to block: %w", err) + } + // insert boundary + err = operation.InsertClusterFinalizedHeight(chainID, genesis.Header.Height)(tx) + // insert started view for hotstuff + if err != nil { + return fmt.Errorf("could not insert genesis boundary: %w", err) + } + + safetyData := &hotstuff.SafetyData{ + LockedOneChainView: genesis.Header.View, + HighestAcknowledgedView: genesis.Header.View, + } + + livenessData := &hotstuff.LivenessData{ + CurrentView: genesis.Header.View + 1, + NewestQC: rootQC, + } + // insert safety data + err = operation.InsertSafetyData(chainID, safetyData)(tx) + if err != nil { + return fmt.Errorf("could not insert safety data: %w", err) + } + // insert liveness data + err = operation.InsertLivenessData(chainID, livenessData)(tx) + if err != nil { + return fmt.Errorf("could not insert liveness data: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("bootstrapping failed: %w", err) + } + + return state, nil +} + +func OpenState(db *badger.DB, _ module.Tracer, _ storage.Headers, _ storage.ClusterPayloads, clusterID flow.ChainID, epoch uint64) (*State, error) { + isBootstrapped, err := IsBootstrapped(db, clusterID) + if err != nil { + return nil, fmt.Errorf("failed to determine whether database contains bootstrapped state: %w", err) + } + if !isBootstrapped { + return nil, fmt.Errorf("expected database to contain bootstrapped state") + } + state := newState(db, clusterID, epoch) + return state, nil +} + +func newState(db *badger.DB, clusterID flow.ChainID, epoch uint64) *State { + state := &State{ + db: db, + clusterID: clusterID, + epoch: epoch, + } + return state +} + +func (s *State) Params() cluster.Params { + params := &Params{ + state: s, + } + return params +} + +func (s *State) Final() cluster.Snapshot { + // get the finalized block ID + var blockID flow.Identifier + err := s.db.View(func(tx *badger.Txn) error { + var boundary uint64 + err := operation.RetrieveClusterFinalizedHeight(s.clusterID, &boundary)(tx) + if err != nil { + return fmt.Errorf("could not retrieve finalized boundary: %w", err) + } + + err = operation.LookupClusterBlockHeight(s.clusterID, boundary, &blockID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve finalized ID: %w", err) + } + + return nil + }) + if err != nil { + return &Snapshot{ + err: err, + } + } + + snapshot := &Snapshot{ + state: s, + blockID: blockID, + } + return snapshot +} + +func (s *State) AtBlockID(blockID flow.Identifier) cluster.Snapshot { + snapshot := &Snapshot{ + state: s, + blockID: blockID, + } + return snapshot +} + +// IsBootstrapped returns whether the database contains a bootstrapped state. +func IsBootstrapped(db *badger.DB, clusterID flow.ChainID) (bool, error) { + var finalized uint64 + err := db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &finalized)) + if errors.Is(err, storage.ErrNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("retrieving finalized height failed: %w", err) + } + return true, nil +} diff --git a/state/cluster/pebble/state_root.go b/state/cluster/pebble/state_root.go new file mode 100644 index 00000000000..50f15d0a373 --- /dev/null +++ b/state/cluster/pebble/state_root.go @@ -0,0 +1,67 @@ +package badger + +import ( + "fmt" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" +) + +// StateRoot is the root information required to bootstrap the cluster state. +type StateRoot struct { + block *cluster.Block // root block for the cluster chain + qc *flow.QuorumCertificate // root QC for the cluster chain + epoch uint64 // operating epoch for the cluster chain +} + +func NewStateRoot(genesis *cluster.Block, qc *flow.QuorumCertificate, epoch uint64) (*StateRoot, error) { + err := validateClusterGenesis(genesis) + if err != nil { + return nil, fmt.Errorf("inconsistent state root: %w", err) + } + return &StateRoot{ + block: genesis, + qc: qc, + epoch: epoch, + }, nil +} + +func validateClusterGenesis(genesis *cluster.Block) error { + // check height of genesis block + if genesis.Header.Height != 0 { + return fmt.Errorf("height of genesis cluster block should be 0 (got %d)", genesis.Header.Height) + } + // check header parent ID + if genesis.Header.ParentID != flow.ZeroID { + return fmt.Errorf("genesis parent ID must be zero hash (got %x)", genesis.Header.ParentID) + } + + // check payload integrity + if genesis.Header.PayloadHash != genesis.Payload.Hash() { + return fmt.Errorf("computed payload hash does not match header") + } + + // check payload + collSize := len(genesis.Payload.Collection.Transactions) + if collSize != 0 { + return fmt.Errorf("genesis collection should contain no transactions (got %d)", collSize) + } + + return nil +} + +func (s StateRoot) ClusterID() flow.ChainID { + return s.block.Header.ChainID +} + +func (s StateRoot) Block() *cluster.Block { + return s.block +} + +func (s StateRoot) QC() *flow.QuorumCertificate { + return s.qc +} + +func (s StateRoot) EpochCounter() uint64 { + return s.epoch +} diff --git a/state/cluster/pebble/translator.go b/state/cluster/pebble/translator.go new file mode 100644 index 00000000000..a7c5269d68f --- /dev/null +++ b/state/cluster/pebble/translator.go @@ -0,0 +1,55 @@ +package badger + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// Translator is a translation layer that determines the reference block on +// the main chain for a given cluster block, using the reference block from +// the cluster block's payload. +type Translator struct { + payloads storage.ClusterPayloads + state protocol.State +} + +// NewTranslator returns a new block ID translator. +func NewTranslator(payloads storage.ClusterPayloads, state protocol.State) *Translator { + translator := &Translator{ + payloads: payloads, + state: state, + } + return translator +} + +// Translate retrieves the reference main-chain block ID for the given cluster +// block ID. +func (t *Translator) Translate(blockID flow.Identifier) (flow.Identifier, error) { + + payload, err := t.payloads.ByBlockID(blockID) + if err != nil { + return flow.ZeroID, fmt.Errorf("could not retrieve reference block payload: %w", err) + } + + // if a reference block is specified, use that + if payload.ReferenceBlockID != flow.ZeroID { + return payload.ReferenceBlockID, nil + } + + // otherwise, we are dealing with a root block, and must retrieve the + // reference block by epoch number + //TODO this returns the latest block in the epoch, thus will take slashing + // into account. We don't slash yet, so this is OK short-term. + // We should change the API boundaries a bit here, so this chain-aware + // translation changes to be f(blockID) -> IdentityList rather than + // f(blockID) -> blockID. + // REF: https://github.com/dapperlabs/flow-go/issues/4655 + head, err := t.state.Final().Head() + if err != nil { + return flow.ZeroID, fmt.Errorf("could not retrieve block: %w", err) + } + return head.ID(), nil +} diff --git a/state/protocol/pebble/mutator.go b/state/protocol/pebble/mutator.go new file mode 100644 index 00000000000..dd2f2035656 --- /dev/null +++ b/state/protocol/pebble/mutator.go @@ -0,0 +1,1208 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "context" + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// FollowerState implements a lighter version of a mutable protocol state. +// When extending the state, it performs hardly any checks on the block payload. +// Instead, the FollowerState relies on the consensus nodes to run the full +// payload check and uses quorum certificates to prove validity of block payloads. +// Consequently, a block B should only be considered valid, if +// there is a certifying QC for that block QC.View == Block.View && QC.BlockID == Block.ID(). +// +// The FollowerState allows non-consensus nodes to execute fork-aware queries +// against the protocol state, while minimizing the amount of payload checks +// the non-consensus nodes have to perform. +type FollowerState struct { + *State + + index storage.Index + payloads storage.Payloads + tracer module.Tracer + logger zerolog.Logger + consumer protocol.Consumer + blockTimer protocol.BlockTimer +} + +var _ protocol.FollowerState = (*FollowerState)(nil) + +// ParticipantState implements a mutable state for consensus participant. It can extend the +// state with a new block, by checking the _entire_ block payload. +type ParticipantState struct { + *FollowerState + receiptValidator module.ReceiptValidator + sealValidator module.SealValidator +} + +var _ protocol.ParticipantState = (*ParticipantState)(nil) + +// NewFollowerState initializes a light-weight version of a mutable protocol +// state. This implementation is suitable only for NON-Consensus nodes. +func NewFollowerState( + logger zerolog.Logger, + tracer module.Tracer, + consumer protocol.Consumer, + state *State, + index storage.Index, + payloads storage.Payloads, + blockTimer protocol.BlockTimer, +) (*FollowerState, error) { + followerState := &FollowerState{ + State: state, + index: index, + payloads: payloads, + tracer: tracer, + logger: logger, + consumer: consumer, + blockTimer: blockTimer, + } + return followerState, nil +} + +// NewFullConsensusState initializes a new mutable protocol state backed by a +// badger database. When extending the state with a new block, it checks the +// _entire_ block payload. Consensus nodes should use the FullConsensusState, +// while other node roles can use the lighter FollowerState. +func NewFullConsensusState( + logger zerolog.Logger, + tracer module.Tracer, + consumer protocol.Consumer, + state *State, + index storage.Index, + payloads storage.Payloads, + blockTimer protocol.BlockTimer, + receiptValidator module.ReceiptValidator, + sealValidator module.SealValidator, +) (*ParticipantState, error) { + followerState, err := NewFollowerState( + logger, + tracer, + consumer, + state, + index, + payloads, + blockTimer, + ) + if err != nil { + return nil, fmt.Errorf("initialization of Mutable Follower State failed: %w", err) + } + return &ParticipantState{ + FollowerState: followerState, + receiptValidator: receiptValidator, + sealValidator: sealValidator, + }, nil +} + +// ExtendCertified extends the protocol state of a CONSENSUS FOLLOWER. While it checks +// the validity of the header; it does _not_ check the validity of the payload. +// Instead, the consensus follower relies on the consensus participants to +// validate the full payload. Payload validity can be proved by a valid quorum certificate. +// Certifying QC must match candidate block: +// +// candidate.View == certifyingQC.View && candidate.ID() == certifyingQC.BlockID +// +// Caution: +// - This function expects that `certifyingQC` has been validated. +// - The parent block must already be stored. +// +// No errors are expected during normal operations. +func (m *FollowerState) ExtendCertified(ctx context.Context, candidate *flow.Block, certifyingQC *flow.QuorumCertificate) error { + span, ctx := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorHeaderExtend) + defer span.End() + + // check if candidate block has been already processed + blockID := candidate.ID() + isDuplicate, err := m.checkBlockAlreadyProcessed(blockID) + if err != nil || isDuplicate { + return err + } + + // sanity check if certifyingQC actually certifies candidate block + if certifyingQC.View != candidate.Header.View { + return fmt.Errorf("qc doesn't certify candidate block, expect %d view, got %d", candidate.Header.View, certifyingQC.View) + } + if certifyingQC.BlockID != blockID { + return fmt.Errorf("qc doesn't certify candidate block, expect %x blockID, got %x", blockID, certifyingQC.BlockID) + } + + // check if the block header is a valid extension of parent block + err = m.headerExtend(candidate) + if err != nil { + // since we have a QC for this block, it cannot be an invalid extension + return fmt.Errorf("unexpected invalid block (id=%x) with certifying qc (id=%x): %s", + candidate.ID(), certifyingQC.ID(), err.Error()) + } + + // find the last seal at the parent block + last, err := m.lastSealed(candidate) + if err != nil { + return fmt.Errorf("payload seal(s) not compliant with chain state: %w", err) + } + + // insert the block, certifying QC and index the last seal for the block + err = m.insert(ctx, candidate, certifyingQC, last) + if err != nil { + return fmt.Errorf("failed to insert the block: %w", err) + } + + return nil +} + +// Extend extends the protocol state of a CONSENSUS PARTICIPANT. It checks +// the validity of the _entire block_ (header and full payload). +// Expected errors during normal operations: +// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) +// - state.InvalidExtensionError if the candidate block is invalid +func (m *ParticipantState) Extend(ctx context.Context, candidate *flow.Block) error { + span, ctx := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtend) + defer span.End() + + // check if candidate block has been already processed + isDuplicate, err := m.checkBlockAlreadyProcessed(candidate.ID()) + if err != nil || isDuplicate { + return err + } + + // check if the block header is a valid extension of parent block + err = m.headerExtend(candidate) + if err != nil { + return fmt.Errorf("header not compliant with chain state: %w", err) + } + + // check if the block header is a valid extension of the finalized state + err = m.checkOutdatedExtension(candidate.Header) + if err != nil { + if state.IsOutdatedExtensionError(err) { + return fmt.Errorf("candidate block is an outdated extension: %w", err) + } + return fmt.Errorf("could not check if block is an outdated extension: %w", err) + } + + // check if the guarantees in the payload is a valid extension of the finalized state + err = m.guaranteeExtend(ctx, candidate) + if err != nil { + return fmt.Errorf("payload guarantee(s) not compliant with chain state: %w", err) + } + + // check if the receipts in the payload are valid + err = m.receiptExtend(ctx, candidate) + if err != nil { + return fmt.Errorf("payload receipt(s) not compliant with chain state: %w", err) + } + + // check if the seals in the payload is a valid extension of the finalized state + lastSeal, err := m.sealExtend(ctx, candidate) + if err != nil { + return fmt.Errorf("payload seal(s) not compliant with chain state: %w", err) + } + + // insert the block and index the last seal for the block + err = m.insert(ctx, candidate, nil, lastSeal) + if err != nil { + return fmt.Errorf("failed to insert the block: %w", err) + } + + return nil +} + +// headerExtend verifies the validity of the block header (excluding verification of the +// consensus rules). Specifically, we check that the block connects to the last finalized block. +// Expected errors during normal operations: +// - state.InvalidExtensionError if the candidate block is invalid +func (m *FollowerState) headerExtend(candidate *flow.Block) error { + // FIRST: We do some initial cheap sanity checks, like checking the payload + // hash is consistent + + header := candidate.Header + payload := candidate.Payload + if payload.Hash() != header.PayloadHash { + return state.NewInvalidExtensionError("payload integrity check failed") + } + + // SECOND: Next, we can check whether the block is a valid descendant of the + // parent. It should have the same chain ID and a height that is one bigger. + + parent, err := m.headers.ByBlockID(header.ParentID) + if err != nil { + return state.NewInvalidExtensionErrorf("could not retrieve parent: %s", err) + } + if header.ChainID != parent.ChainID { + return state.NewInvalidExtensionErrorf("candidate built for invalid chain (candidate: %s, parent: %s)", + header.ChainID, parent.ChainID) + } + if header.ParentView != parent.View { + return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", + header.ParentView, parent.View) + } + if header.Height != parent.Height+1 { + return state.NewInvalidExtensionErrorf("candidate built with invalid height (candidate: %d, parent: %d)", + header.Height, parent.Height) + } + + // check validity of block timestamp using parent's timestamp + err = m.blockTimer.Validate(parent.Timestamp, candidate.Header.Timestamp) + if err != nil { + if protocol.IsInvalidBlockTimestampError(err) { + return state.NewInvalidExtensionErrorf("candidate contains invalid timestamp: %w", err) + } + return fmt.Errorf("validating block's time stamp failed with unexpected error: %w", err) + } + + return nil +} + +// checkBlockAlreadyProcessed checks if block has been added to the protocol state. +// Returns: +// * (true, nil) - block has been already processed. +// * (false, nil) - block has not been processed. +// * (false, error) - unknown error when trying to query protocol state. +// No errors are expected during normal operation. +func (m *FollowerState) checkBlockAlreadyProcessed(blockID flow.Identifier) (bool, error) { + _, err := m.headers.ByBlockID(blockID) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("could not check if candidate block (%x) has been already processed: %w", blockID, err) + } + return true, nil +} + +// checkOutdatedExtension checks whether given block is +// valid in the context of the entire state. For this, the block needs to +// directly connect, through its ancestors, to the last finalized block. +// Expected errors during normal operations: +// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) +func (m *ParticipantState) checkOutdatedExtension(header *flow.Header) error { + var finalizedHeight uint64 + err := m.db.View(operation.RetrieveFinalizedHeight(&finalizedHeight)) + if err != nil { + return fmt.Errorf("could not retrieve finalized height: %w", err) + } + var finalID flow.Identifier + err = m.db.View(operation.LookupBlockHeight(finalizedHeight, &finalID)) + if err != nil { + return fmt.Errorf("could not lookup finalized block: %w", err) + } + + ancestorID := header.ParentID + for ancestorID != finalID { + ancestor, err := m.headers.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not retrieve ancestor (%x): %w", ancestorID, err) + } + if ancestor.Height < finalizedHeight { + // this happens when the candidate block is on a fork that does not include all the + // finalized blocks. + // for instance: + // A (Finalized) <- B (Finalized) <- C (Finalized) <- D <- E <- F + // ^- G ^- H ^- I + // block G is not a valid block, because it does not have C (which has been finalized) as an ancestor + // block H and I are valid, because they do have C as an ancestor + return state.NewOutdatedExtensionErrorf( + "candidate block (height: %d) conflicts with finalized state (ancestor: %d final: %d)", + header.Height, ancestor.Height, finalizedHeight) + } + ancestorID = ancestor.ParentID + } + return nil +} + +// guaranteeExtend verifies the validity of the collection guarantees that are +// included in the block. Specifically, we check for expired collections and +// duplicated collections (also including ancestor blocks). +func (m *ParticipantState) guaranteeExtend(ctx context.Context, candidate *flow.Block) error { + + span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtendCheckGuarantees) + defer span.End() + + header := candidate.Header + payload := candidate.Payload + + // we only look as far back for duplicates as the transaction expiry limit; + // if a guarantee was included before that, we will disqualify it on the + // basis of the reference block anyway + limit := header.Height - flow.DefaultTransactionExpiry + if limit > header.Height { // overflow check + limit = 0 + } + if limit < m.sporkRootBlockHeight { + limit = m.sporkRootBlockHeight + } + + // build a list of all previously used guarantees on this part of the chain + ancestorID := header.ParentID + lookup := make(map[flow.Identifier]struct{}) + for { + ancestor, err := m.headers.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not retrieve ancestor header (%x): %w", ancestorID, err) + } + index, err := m.index.ByBlockID(ancestorID) + if err != nil { + return fmt.Errorf("could not retrieve ancestor index (%x): %w", ancestorID, err) + } + for _, collID := range index.CollectionIDs { + lookup[collID] = struct{}{} + } + if ancestor.Height <= limit { + break + } + ancestorID = ancestor.ParentID + } + + // check each guarantee included in the payload for duplication and expiry + for _, guarantee := range payload.Guarantees { + + // if the guarantee was already included before, error + _, duplicated := lookup[guarantee.ID()] + if duplicated { + return state.NewInvalidExtensionErrorf("payload includes duplicate guarantee (%x)", guarantee.ID()) + } + + // get the reference block to check expiry + ref, err := m.headers.ByBlockID(guarantee.ReferenceBlockID) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return state.NewInvalidExtensionErrorf("could not get reference block %x: %w", guarantee.ReferenceBlockID, err) + } + return fmt.Errorf("could not get reference block (%x): %w", guarantee.ReferenceBlockID, err) + } + + // if the guarantee references a block with expired height, error + if ref.Height < limit { + return state.NewInvalidExtensionErrorf("payload includes expired guarantee (height: %d, limit: %d)", + ref.Height, limit) + } + + // check the guarantors are correct + _, err = protocol.FindGuarantors(m, guarantee) + if err != nil { + if signature.IsInvalidSignerIndicesError(err) || + errors.Is(err, protocol.ErrNextEpochNotCommitted) || + errors.Is(err, protocol.ErrClusterNotFound) { + return state.NewInvalidExtensionErrorf("guarantee %v contains invalid guarantors: %w", guarantee.ID(), err) + } + return fmt.Errorf("could not find guarantor for guarantee %v: %w", guarantee.ID(), err) + } + } + + return nil +} + +// sealExtend checks the compliance of the payload seals. Returns last seal that form a chain for +// candidate block. +func (m *ParticipantState) sealExtend(ctx context.Context, candidate *flow.Block) (*flow.Seal, error) { + + span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtendCheckSeals) + defer span.End() + + lastSeal, err := m.sealValidator.Validate(candidate) + if err != nil { + return nil, state.NewInvalidExtensionErrorf("seal validation error: %w", err) + } + + return lastSeal, nil +} + +// receiptExtend checks the compliance of the receipt payload. +// - Receipts should pertain to blocks on the fork +// - Receipts should not appear more than once on a fork +// - Receipts should pass the ReceiptValidator check +// - No seal has been included for the respective block in this particular fork +// +// We require the receipts to be sorted by block height (within a payload). +func (m *ParticipantState) receiptExtend(ctx context.Context, candidate *flow.Block) error { + + span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtendCheckReceipts) + defer span.End() + + err := m.receiptValidator.ValidatePayload(candidate) + if err != nil { + // TODO: this might be not an error, potentially it can be solved by requesting more data and processing this receipt again + if errors.Is(err, storage.ErrNotFound) { + return state.NewInvalidExtensionErrorf("some entities referenced by receipts are missing: %w", err) + } + if engine.IsInvalidInputError(err) { + return state.NewInvalidExtensionErrorf("payload includes invalid receipts: %w", err) + } + return fmt.Errorf("unexpected payload validation error %w", err) + } + + return nil +} + +// lastSealed returns the highest sealed block from the fork with head `candidate`. +// For instance, here is the chain state: block 100 is the head, block 97 is finalized, +// and 95 is the last sealed block at the state of block 100. +// 95 (sealed) <- 96 <- 97 (finalized) <- 98 <- 99 <- 100 +// Now, if block 101 is extending block 100, and its payload has a seal for 96, then it will +// be the last sealed for block 101. +// No errors are expected during normal operation. +func (m *FollowerState) lastSealed(candidate *flow.Block) (*flow.Seal, error) { + header := candidate.Header + payload := candidate.Payload + + // getting the last sealed block + last, err := m.seals.HighestInFork(header.ParentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve parent seal (%x): %w", header.ParentID, err) + } + + // if the payload of the block has no seals, then the last seal is the seal for the highest block + if len(payload.Seals) == 0 { + return last, nil + } + + ordered, err := protocol.OrderedSeals(payload, m.headers) + if err != nil { + // all errors are unexpected - differentiation is for clearer error messages + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("ordering seals: candidate payload contains seals for unknown block: %s", err.Error()) + } + if errors.Is(err, protocol.ErrDiscontinuousSeals) || errors.Is(err, protocol.ErrMultipleSealsForSameHeight) { + return nil, fmt.Errorf("ordering seals: candidate payload contains invalid seal set: %s", err.Error()) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + return ordered[len(ordered)-1], nil +} + +// insert stores the candidate block in the database. +// The `candidate` block _must be valid_ (otherwise, the state will be corrupted). +// dbUpdates contains other database operations which must be applied atomically +// with inserting the block. +// Caller is responsible for ensuring block validity. +// If insert is called from Extend(by consensus participant) then certifyingQC will be nil but the block payload will be validated. +// If insert is called from ExtendCertified(by consensus follower) then certifyingQC must be not nil which proves payload validity. +// No errors are expected during normal operations. +func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certifyingQC *flow.QuorumCertificate, last *flow.Seal) error { + span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtendDBInsert) + defer span.End() + + blockID := candidate.ID() + parentID := candidate.Header.ParentID + latestSealID := last.ID() + + parent, err := m.headers.ByBlockID(parentID) + if err != nil { + return fmt.Errorf("could not retrieve block header for %x: %w", parentID, err) + } + + // apply any state changes from service events sealed by this block's parent + dbUpdates, err := m.handleEpochServiceEvents(candidate) + if err != nil { + return fmt.Errorf("could not process service events: %w", err) + } + + qc := candidate.Header.QuorumCertificate() + + var events []func() + + // Both the header itself and its payload are in compliance with the protocol state. + // We can now store the candidate block, as well as adding its final seal + // to the seal index and initializing its children index. + err = operation.RetryOnConflictTx(m.db, transaction.Update, func(tx *transaction.Tx) error { + // insert the block into the database AND cache + err := m.blocks.StoreTx(candidate)(tx) + if err != nil { + return fmt.Errorf("could not store candidate block: %w", err) + } + + err = m.qcs.StoreTx(qc)(tx) + if err != nil { + if !errors.Is(err, storage.ErrAlreadyExists) { + return fmt.Errorf("could not store incorporated qc: %w", err) + } + } else { + // trigger BlockProcessable for parent blocks above root height + if parent.Height > m.finalizedRootHeight { + events = append(events, func() { + m.consumer.BlockProcessable(parent, qc) + }) + } + } + + if certifyingQC != nil { + err = m.qcs.StoreTx(certifyingQC)(tx) + if err != nil { + return fmt.Errorf("could not store certifying qc: %w", err) + } + + // trigger BlockProcessable for candidate block if it's certified + events = append(events, func() { + m.consumer.BlockProcessable(candidate.Header, certifyingQC) + }) + } + + // index the latest sealed block in this fork + err = transaction.WithTx(operation.IndexLatestSealAtBlock(blockID, latestSealID))(tx) + if err != nil { + return fmt.Errorf("could not index candidate seal: %w", err) + } + + // index the child block for recovery + err = transaction.WithTx(procedure.IndexNewBlock(blockID, candidate.Header.ParentID))(tx) + if err != nil { + return fmt.Errorf("could not index new block: %w", err) + } + + // apply any optional DB operations from service events + for _, apply := range dbUpdates { + err := apply(tx) + if err != nil { + return fmt.Errorf("could not apply operation: %w", err) + } + } + + return nil + }) + if err != nil { + return fmt.Errorf("could not execute state extension: %w", err) + } + + // execute scheduled events + for _, event := range events { + event() + } + + return nil +} + +// Finalize marks the specified block as finalized. +// This method only finalizes one block at a time. +// Hence, the parent of `blockID` has to be the last finalized block. +// No errors are expected during normal operations. +func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) error { + + // preliminaries: start tracer and retrieve full block + span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorFinalize) + defer span.End() + block, err := m.blocks.ByID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve full block that should be finalized: %w", err) + } + header := block.Header + + // keep track of metrics updates and protocol events to emit: + // * metrics are updated after a successful database update + // * protocol events are emitted atomically with the database update + var metrics []func() + var events []func() + + // Verify that the parent block is the latest finalized block. + // this must be the case, as the `Finalize` method only finalizes one block + // at a time and hence the parent of `blockID` must already be finalized. + var finalized uint64 + err = m.db.View(operation.RetrieveFinalizedHeight(&finalized)) + if err != nil { + return fmt.Errorf("could not retrieve finalized height: %w", err) + } + var finalID flow.Identifier + err = m.db.View(operation.LookupBlockHeight(finalized, &finalID)) + if err != nil { + return fmt.Errorf("could not retrieve final header: %w", err) + } + if header.ParentID != finalID { + return fmt.Errorf("can only finalize child of last finalized block") + } + + // We also want to update the last sealed height. Retrieve the block + // seal indexed for the block and retrieve the block that was sealed by it. + lastSeal, err := m.seals.HighestInFork(blockID) + if err != nil { + return fmt.Errorf("could not look up sealed header: %w", err) + } + sealed, err := m.headers.ByBlockID(lastSeal.BlockID) + if err != nil { + return fmt.Errorf("could not retrieve sealed header: %w", err) + } + + // We update metrics and emit protocol events for epoch state changes when + // the block corresponding to the state change is finalized + epochStatus, err := m.epoch.statuses.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve epoch state: %w", err) + } + currentEpochSetup, err := m.epoch.setups.ByID(epochStatus.CurrentEpoch.SetupID) + if err != nil { + return fmt.Errorf("could not retrieve setup event for current epoch: %w", err) + } + epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() + if err != nil { + return fmt.Errorf("could not check persisted epoch emergency fallback flag: %w", err) + } + + // if epoch fallback was not previously triggered, check whether this block triggers it + if !epochFallbackTriggered { + epochFallbackTriggered, err = m.epochFallbackTriggeredByFinalizedBlock(header, epochStatus, currentEpochSetup) + if err != nil { + return fmt.Errorf("could not check whether finalized block triggers epoch fallback: %w", err) + } + if epochFallbackTriggered { + // emit the protocol event only the first time epoch fallback is triggered + events = append(events, m.consumer.EpochEmergencyFallbackTriggered) + metrics = append(metrics, m.metrics.EpochEmergencyFallbackTriggered) + } + } + + isFirstBlockOfEpoch, err := m.isFirstBlockOfEpoch(header, currentEpochSetup) + if err != nil { + return fmt.Errorf("could not check if block is first of epoch: %w", err) + } + + // Determine metric updates and protocol events related to epoch phase + // changes and epoch transitions. + // If epoch emergency fallback is triggered, the current epoch continues until + // the next spork - so skip these updates. + if !epochFallbackTriggered { + epochPhaseMetrics, epochPhaseEvents, err := m.epochPhaseMetricsAndEventsOnBlockFinalized(block, epochStatus) + if err != nil { + return fmt.Errorf("could not determine epoch phase metrics/events for finalized block: %w", err) + } + metrics = append(metrics, epochPhaseMetrics...) + events = append(events, epochPhaseEvents...) + + if isFirstBlockOfEpoch { + epochTransitionMetrics, epochTransitionEvents := m.epochTransitionMetricsAndEventsOnBlockFinalized(header, currentEpochSetup) + if err != nil { + return fmt.Errorf("could not determine epoch transition metrics/events for finalized block: %w", err) + } + metrics = append(metrics, epochTransitionMetrics...) + events = append(events, epochTransitionEvents...) + } + } + + // Extract and validate version beacon events from the block seals. + versionBeacons, err := m.versionBeaconOnBlockFinalized(block) + if err != nil { + return fmt.Errorf("cannot process version beacon: %w", err) + } + + // Persist updates in database + // * Add this block to the height-indexed set of finalized blocks. + // * Update the largest finalized height to this block's height. + // * Update the largest height of sealed and finalized block. + // This value could actually stay the same if it has no seals in + // its payload, in which case the parent's seal is the same. + // * set the epoch fallback flag, if it is triggered + err = operation.RetryOnConflict(m.db.Update, func(tx *badger.Txn) error { + err = operation.IndexBlockHeight(header.Height, blockID)(tx) + if err != nil { + return fmt.Errorf("could not insert number mapping: %w", err) + } + err = operation.UpdateFinalizedHeight(header.Height)(tx) + if err != nil { + return fmt.Errorf("could not update finalized height: %w", err) + } + err = operation.UpdateSealedHeight(sealed.Height)(tx) + if err != nil { + return fmt.Errorf("could not update sealed height: %w", err) + } + if epochFallbackTriggered { + err = operation.SetEpochEmergencyFallbackTriggered(blockID)(tx) + if err != nil { + return fmt.Errorf("could not set epoch fallback flag: %w", err) + } + } + if isFirstBlockOfEpoch && !epochFallbackTriggered { + err = operation.InsertEpochFirstHeight(currentEpochSetup.Counter, header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert epoch first block height: %w", err) + } + } + + // When a block is finalized, we commit the result for each seal it contains. The sealing logic + // guarantees that only a single, continuous execution fork is sealed. Here, we index for + // each block ID the ID of its _finalized_ seal. + for _, seal := range block.Payload.Seals { + err = operation.IndexFinalizedSealByBlockID(seal.BlockID, seal.ID())(tx) + if err != nil { + return fmt.Errorf("could not index the seal by the sealed block ID: %w", err) + } + } + + if len(versionBeacons) > 0 { + // only index the last version beacon as that is the relevant one. + // TODO: The other version beacons can be used for validation. + err := operation.IndexVersionBeaconByHeight(versionBeacons[len(versionBeacons)-1])(tx) + if err != nil { + return fmt.Errorf("could not index version beacon or height (%d): %w", header.Height, err) + } + } + + return nil + }) + if err != nil { + return fmt.Errorf("could not persist finalization operations for block (%x): %w", blockID, err) + } + + // update the cache + m.State.cachedFinal.Store(&cachedHeader{blockID, header}) + if len(block.Payload.Seals) > 0 { + m.State.cachedSealed.Store(&cachedHeader{lastSeal.BlockID, sealed}) + } + + // Emit protocol events after database transaction succeeds. Event delivery is guaranteed, + // _except_ in case of a crash. Hence, when recovering from a crash, consumers need to deduce + // from the state whether they have missed events and re-execute the respective actions. + m.consumer.BlockFinalized(header) + for _, emit := range events { + emit() + } + + // update sealed/finalized block metrics + m.metrics.FinalizedHeight(header.Height) + m.metrics.SealedHeight(sealed.Height) + m.metrics.BlockFinalized(block) + for _, seal := range block.Payload.Seals { + sealedBlock, err := m.blocks.ByID(seal.BlockID) + if err != nil { + return fmt.Errorf("could not retrieve sealed block (%x): %w", seal.BlockID, err) + } + m.metrics.BlockSealed(sealedBlock) + } + + // apply all queued metrics + for _, updateMetric := range metrics { + updateMetric() + } + + return nil +} + +// epochFallbackTriggeredByFinalizedBlock checks whether finalizing the input block +// would trigger epoch emergency fallback mode. In particular, we trigger epoch +// fallback mode while finalizing block B in either of the following cases: +// 1. B is the head of a fork in which epoch fallback was tentatively triggered, +// due to incorporating an invalid service event. +// 2. (a) B is the first finalized block with view greater than or equal to the epoch +// commitment deadline for the current epoch AND +// (b) the next epoch has not been committed as of B. +// +// This function should only be called when epoch fallback *has not already been triggered*. +// See protocol.Params for more details on the epoch commitment deadline. +// +// No errors are expected during normal operation. +func (m *FollowerState) epochFallbackTriggeredByFinalizedBlock(block *flow.Header, epochStatus *flow.EpochStatus, currentEpochSetup *flow.EpochSetup) (bool, error) { + // 1. Epoch fallback is tentatively triggered on this fork + if epochStatus.InvalidServiceEventIncorporated { + return true, nil + } + + // 2.(a) determine whether block B is past the epoch commitment deadline + safetyThreshold, err := m.Params().EpochCommitSafetyThreshold() + if err != nil { + return false, fmt.Errorf("could not get epoch commit safety threshold: %w", err) + } + blockExceedsDeadline := block.View+safetyThreshold >= currentEpochSetup.FinalView + + // 2.(b) determine whether the next epoch is committed w.r.t. block B + currentEpochPhase, err := epochStatus.Phase() + if err != nil { + return false, fmt.Errorf("could not get current epoch phase: %w", err) + } + isNextEpochCommitted := currentEpochPhase == flow.EpochPhaseCommitted + + blockTriggersEpochFallback := blockExceedsDeadline && !isNextEpochCommitted + return blockTriggersEpochFallback, nil +} + +// isFirstBlockOfEpoch returns true if the given block is the first block of a new epoch. +// We accept the EpochSetup event for the current epoch (w.r.t. input block B) which contains +// the FirstView for the epoch (denoted W). By construction, B.View >= W. +// Definition: B is the first block of the epoch if and only if B.parent.View < W +// +// NOTE: There can be multiple (un-finalized) blocks that qualify as the first block of epoch N. +// No errors are expected during normal operation. +func (m *FollowerState) isFirstBlockOfEpoch(block *flow.Header, currentEpochSetup *flow.EpochSetup) (bool, error) { + currentEpochFirstView := currentEpochSetup.FirstView + // sanity check: B.View >= W + if block.View < currentEpochFirstView { + return false, irrecoverable.NewExceptionf("data inconsistency: block (id=%x, view=%d) is below its epoch first view %d", block.ID(), block.View, currentEpochFirstView) + } + + parent, err := m.headers.ByBlockID(block.ParentID) + if err != nil { + return false, irrecoverable.NewExceptionf("could not retrieve parent (id=%s): %w", block.ParentID, err) + } + + return parent.View < currentEpochFirstView, nil +} + +// epochTransitionMetricsAndEventsOnBlockFinalized determines metrics to update +// and protocol events to emit for blocks which are the first block of a new epoch. +// Protocol events and updating metrics happen once when we finalize the _first_ +// block of the new Epoch (same convention as for Epoch-Phase-Changes). +// +// NOTE: This function must only be called when input `block` is the first block +// of the epoch denoted by `currentEpochSetup`. +func (m *FollowerState) epochTransitionMetricsAndEventsOnBlockFinalized(block *flow.Header, currentEpochSetup *flow.EpochSetup) ( + metrics []func(), + events []func(), +) { + + events = append(events, func() { m.consumer.EpochTransition(currentEpochSetup.Counter, block) }) + // set current epoch counter corresponding to new epoch + metrics = append(metrics, func() { m.metrics.CurrentEpochCounter(currentEpochSetup.Counter) }) + // denote the most recent epoch transition height + metrics = append(metrics, func() { m.metrics.EpochTransitionHeight(block.Height) }) + // set epoch phase - since we are starting a new epoch we begin in the staking phase + metrics = append(metrics, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseStaking) }) + // set current epoch view values + metrics = append( + metrics, + func() { m.metrics.CurrentEpochFinalView(currentEpochSetup.FinalView) }, + func() { m.metrics.CurrentDKGPhase1FinalView(currentEpochSetup.DKGPhase1FinalView) }, + func() { m.metrics.CurrentDKGPhase2FinalView(currentEpochSetup.DKGPhase2FinalView) }, + func() { m.metrics.CurrentDKGPhase3FinalView(currentEpochSetup.DKGPhase3FinalView) }, + ) + + return +} + +// epochPhaseMetricsAndEventsOnBlockFinalized determines metrics to update and protocol +// events to emit. Service Events embedded into an execution result take effect, when the +// execution result's _seal is finalized_ (i.e. when the block holding a seal for the +// result is finalized). See also handleEpochServiceEvents for further details. Example: +// +// Convention: +// +// A <-- ... <-- C(Seal_A) +// +// Suppose an EpochSetup service event is emitted during execution of block A. C seals A, therefore +// we apply the metrics/events when C is finalized. The first block of the EpochSetup +// phase is block C. +// +// This function should only be called when epoch fallback *has not already been triggered*. +// No errors are expected during normal operation. +func (m *FollowerState) epochPhaseMetricsAndEventsOnBlockFinalized(block *flow.Block, epochStatus *flow.EpochStatus) ( + metrics []func(), + events []func(), + err error, +) { + + // block payload may not specify seals in order, so order them by block height before processing + orderedSeals, err := protocol.OrderedSeals(block.Payload, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) + } + return nil, nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + + // track service event driven metrics and protocol events that should be emitted + for _, seal := range orderedSeals { + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, nil, fmt.Errorf("could not retrieve result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + } + for _, event := range result.ServiceEvents { + switch ev := event.Event.(type) { + case *flow.EpochSetup: + // update current epoch phase + events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseSetup) }) + // track epoch phase transition (staking->setup) + events = append(events, func() { m.consumer.EpochSetupPhaseStarted(ev.Counter-1, block.Header) }) + case *flow.EpochCommit: + // update current epoch phase + events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseCommitted) }) + // track epoch phase transition (setup->committed) + events = append(events, func() { m.consumer.EpochCommittedPhaseStarted(ev.Counter-1, block.Header) }) + // track final view of committed epoch + nextEpochSetup, err := m.epoch.setups.ByID(epochStatus.NextEpoch.SetupID) + if err != nil { + return nil, nil, fmt.Errorf("could not retrieve setup event for next epoch: %w", err) + } + events = append(events, func() { m.metrics.CommittedEpochFinalView(nextEpochSetup.FinalView) }) + case *flow.VersionBeacon: + // do nothing for now + default: + return nil, nil, fmt.Errorf("invalid service event type in payload (%T)", ev) + } + } + } + + return +} + +// epochStatus computes the EpochStatus for the given block *before* applying +// any service event state changes which come into effect with this block. +// +// Specifically, we must determine whether block is the first block of a new +// epoch in its respective fork. We do this by comparing the block's view to +// the Epoch data from its parent. If the block's view is _larger_ than the +// final View of the parent's epoch, the block starts a new Epoch. +// +// Possible outcomes: +// 1. Block is in same Epoch as parent (block.View < epoch.FinalView) +// -> the parent's EpochStatus.CurrentEpoch also applies for the current block +// 2. Block enters the next Epoch (block.View ≥ epoch.FinalView) +// a) HAPPY PATH: Epoch fallback is not triggered, we enter the next epoch: +// -> the parent's EpochStatus.NextEpoch is the current block's EpochStatus.CurrentEpoch +// b) FALLBACK PATH: Epoch fallback is triggered, we continue the current epoch: +// -> the parent's EpochStatus.CurrentEpoch also applies for the current block +// +// As the parent was a valid extension of the chain, by induction, the parent +// satisfies all consistency requirements of the protocol. +// +// Returns the EpochStatus for the input block. +// No error returns are expected under normal operations +func (m *FollowerState) epochStatus(block *flow.Header, epochFallbackTriggered bool) (*flow.EpochStatus, error) { + parentStatus, err := m.epoch.statuses.ByBlockID(block.ParentID) + if err != nil { + return nil, fmt.Errorf("could not retrieve epoch state for parent: %w", err) + } + parentSetup, err := m.epoch.setups.ByID(parentStatus.CurrentEpoch.SetupID) + if err != nil { + return nil, fmt.Errorf("could not retrieve EpochSetup event for parent: %w", err) + } + + // Case 1 or 2b (still in parent block's epoch or epoch fallback triggered): + if block.View <= parentSetup.FinalView || epochFallbackTriggered { + // IMPORTANT: copy the status to avoid modifying the parent status in the cache + return parentStatus.Copy(), nil + } + + // Case 2a (first block of new epoch): + // sanity check: parent's epoch Preparation should be completed and have EpochSetup and EpochCommit events + if parentStatus.NextEpoch.SetupID == flow.ZeroID { + return nil, fmt.Errorf("missing setup event for starting next epoch") + } + if parentStatus.NextEpoch.CommitID == flow.ZeroID { + return nil, fmt.Errorf("missing commit event for starting next epoch") + } + epochStatus, err := flow.NewEpochStatus( + parentStatus.CurrentEpoch.SetupID, parentStatus.CurrentEpoch.CommitID, + parentStatus.NextEpoch.SetupID, parentStatus.NextEpoch.CommitID, + flow.ZeroID, flow.ZeroID, + ) + return epochStatus, err + +} + +// versionBeaconOnBlockFinalized extracts and returns the VersionBeacons from the +// finalized block's seals. +// This could return multiple VersionBeacons if the parent block contains multiple Seals. +// The version beacons will be returned in the ascending height order of the seals. +// Technically only the last VersionBeacon is relevant. +func (m *FollowerState) versionBeaconOnBlockFinalized( + finalized *flow.Block, +) ([]*flow.SealedVersionBeacon, error) { + var versionBeacons []*flow.SealedVersionBeacon + + seals, err := protocol.OrderedSeals(finalized.Payload, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf( + "ordering seals: parent payload contains"+ + " seals for unknown block: %w", err) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + + for _, seal := range seals { + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, fmt.Errorf( + "could not retrieve result (id=%x) for seal (id=%x): %w", + seal.ResultID, + seal.ID(), + err) + } + for _, event := range result.ServiceEvents { + + ev, ok := event.Event.(*flow.VersionBeacon) + + if !ok { + // skip other service event types. + // validation if this is a known service event type is done elsewhere. + continue + } + + err := ev.Validate() + if err != nil { + m.logger.Warn(). + Err(err). + Str("block_id", finalized.ID().String()). + Interface("event", ev). + Msg("invalid VersionBeacon service event") + continue + } + + // The version beacon only becomes actionable/valid/active once the block + // containing the version beacon has been sealed. That is why we set the + // Seal height to the current block height. + versionBeacons = append(versionBeacons, &flow.SealedVersionBeacon{ + VersionBeacon: ev, + SealHeight: finalized.Header.Height, + }) + } + } + + return versionBeacons, nil +} + +// handleEpochServiceEvents handles applying state changes which occur as a result +// of service events being included in a block payload: +// - inserting incorporated service events +// - updating EpochStatus for the candidate block +// +// Consider a chain where a service event is emitted during execution of block A. +// Block B contains a receipt for A. Block C contains a seal for block A. +// +// A <- .. <- B(RA) <- .. <- C(SA) +// +// Service events are included within execution results, which are stored +// opaquely as part of the block payload in block B. We only validate and insert +// the typed service event to storage once we process C, the block containing the +// seal for block A. This is because we rely on the sealing subsystem to validate +// correctness of the service event before processing it. +// Consequently, any change to the protocol state introduced by a service event +// emitted during execution of block A would only become visible when querying +// C or its descendants. +// +// This method will only apply service-event-induced state changes when the +// input block has the form of block C (ie. contains a seal for a block in +// which a service event was emitted). +// +// Return values: +// - dbUpdates - If the service events are valid, or there are no service events, +// this method returns a slice of Badger operations to apply while storing the block. +// This includes an operation to index the epoch status for every block, and +// operations to insert service events for blocks that include them. +// +// No errors are expected during normal operation. +func (m *FollowerState) handleEpochServiceEvents(candidate *flow.Block) (dbUpdates []func(*transaction.Tx) error, err error) { + epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + } + epochStatus, err := m.epochStatus(candidate.Header, epochFallbackTriggered) + if err != nil { + return nil, fmt.Errorf("could not determine epoch status for candidate block: %w", err) + } + activeSetup, err := m.epoch.setups.ByID(epochStatus.CurrentEpoch.SetupID) + if err != nil { + return nil, fmt.Errorf("could not retrieve current epoch setup event: %w", err) + } + + // always persist the candidate's epoch status + // note: We are scheduling the operation to store the Epoch status using the _pointer_ variable `epochStatus`. + // The struct `epochStatus` points to will still be modified below. + blockID := candidate.ID() + dbUpdates = append(dbUpdates, m.epoch.statuses.StoreTx(blockID, epochStatus)) + + // never process service events after epoch fallback is triggered + if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { + return dbUpdates, nil + } + + // We apply service events from blocks which are sealed by this candidate block. + // The block's payload might contain epoch preparation service events for the next + // epoch. In this case, we need to update the tentative protocol state. + // We need to validate whether all information is available in the protocol + // state to go to the next epoch when needed. In cases where there is a bug + // in the smart contract, it could be that this happens too late and the + // chain finalization should halt. + + // block payload may not specify seals in order, so order them by block height before processing + orderedSeals, err := protocol.OrderedSeals(candidate.Payload, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + for _, seal := range orderedSeals { + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + } + + for _, event := range result.ServiceEvents { + + switch ev := event.Event.(type) { + case *flow.EpochSetup: + // validate the service event + err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochSetup service event: %w", err) + } + + // prevents multiple setup events for same Epoch (including multiple setup events in payload of same block) + epochStatus.NextEpoch.SetupID = ev.ID() + + // we'll insert the setup event when we insert the block + dbUpdates = append(dbUpdates, m.epoch.setups.StoreTx(ev)) + + case *flow.EpochCommit: + // if we receive an EpochCommit event, we must have already observed an EpochSetup event + // => otherwise, we have observed an EpochCommit without corresponding EpochSetup, which triggers epoch fallback mode + if epochStatus.NextEpoch.SetupID == flow.ZeroID { + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + + // if we have observed an EpochSetup event, we must be able to retrieve it from the database + // => otherwise, this is a symptom of bug or data corruption since this component sets the SetupID field + extendingSetup, err := m.epoch.setups.ByID(epochStatus.NextEpoch.SetupID) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, irrecoverable.NewExceptionf("could not retrieve EpochSetup (id=%x) stored in EpochStatus for block %x: %w", + epochStatus.NextEpoch.SetupID, blockID, err) + } + return nil, fmt.Errorf("unexpected error retrieving next epoch setup: %w", err) + } + + // validate the service event + err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochCommit service event: %w", err) + } + + // prevents multiple setup events for same Epoch (including multiple setup events in payload of same block) + epochStatus.NextEpoch.CommitID = ev.ID() + + // we'll insert the commit event when we insert the block + dbUpdates = append(dbUpdates, m.epoch.commits.StoreTx(ev)) + case *flow.VersionBeacon: + // do nothing for now + default: + return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) + } + } + } + return +} diff --git a/state/protocol/pebble/mutator_test.go b/state/protocol/pebble/mutator_test.go new file mode 100644 index 00000000000..8a63f20aa29 --- /dev/null +++ b/state/protocol/pebble/mutator_test.go @@ -0,0 +1,2548 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger_test + +import ( + "context" + "errors" + "math/rand" + "sync" + "testing" + "time" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module/metrics" + mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/module/trace" + st "github.com/onflow/flow-go/state" + realprotocol "github.com/onflow/flow-go/state/protocol" + protocol "github.com/onflow/flow-go/state/protocol/badger" + "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/state/protocol/inmem" + mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/state/protocol/util" + "github.com/onflow/flow-go/storage" + stoerr "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + storeutil "github.com/onflow/flow-go/storage/util" + "github.com/onflow/flow-go/utils/unittest" +) + +var participants = unittest.IdentityListFixture(5, unittest.WithAllRoles()) + +func TestBootstrapValid(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *protocol.State) { + var finalized uint64 + err := db.View(operation.RetrieveFinalizedHeight(&finalized)) + require.NoError(t, err) + + var sealed uint64 + err = db.View(operation.RetrieveSealedHeight(&sealed)) + require.NoError(t, err) + + var genesisID flow.Identifier + err = db.View(operation.LookupBlockHeight(0, &genesisID)) + require.NoError(t, err) + + var header flow.Header + err = db.View(operation.RetrieveHeader(genesisID, &header)) + require.NoError(t, err) + + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(genesisID, &sealID)) + require.NoError(t, err) + + _, seal, err := rootSnapshot.SealedResult() + require.NoError(t, err) + err = db.View(operation.RetrieveSeal(sealID, seal)) + require.NoError(t, err) + + block, err := rootSnapshot.Head() + require.NoError(t, err) + require.Equal(t, block.Height, finalized) + require.Equal(t, block.Height, sealed) + require.Equal(t, block.ID(), genesisID) + require.Equal(t, block.ID(), seal.BlockID) + require.Equal(t, block, &header) + }) +} + +// TestExtendValid tests the happy path of extending the state with a single block. +// * BlockFinalized is emitted when the block is finalized +// * BlockProcessable is emitted when a block's child is inserted +func TestExtendValid(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := storeutil.StorageLayer(t, db) + + distributor := events.NewDistributor() + consumer := mockprotocol.NewConsumer(t) + distributor.AddConsumer(consumer) + + block, result, seal := unittest.BootstrapFixture(participants) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID())) + rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) + require.NoError(t, err) + + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(t, err) + + fullState, err := protocol.NewFullConsensusState( + log, + tracer, + consumer, + state, + all.Index, + all.Payloads, + util.MockBlockTimer(), + util.MockReceiptValidator(), + util.MockSealValidator(all.Seals), + ) + require.NoError(t, err) + + // insert block1 on top of the root block + block1 := unittest.BlockWithParentFixture(block.Header) + err = fullState.Extend(context.Background(), block1) + require.NoError(t, err) + + // we should not emit BlockProcessable for the root block + consumer.AssertNotCalled(t, "BlockProcessable", block.Header) + + t.Run("BlockFinalized event should be emitted when block1 is finalized", func(t *testing.T) { + consumer.On("BlockFinalized", block1.Header).Once() + err := fullState.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + }) + + t.Run("BlockProcessable event should be emitted when any child of block1 is inserted", func(t *testing.T) { + block2 := unittest.BlockWithParentFixture(block1.Header) + consumer.On("BlockProcessable", block1.Header, mock.Anything).Once() + err := fullState.Extend(context.Background(), block2) + require.NoError(t, err) + }) + }) +} + +func TestSealedIndex(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + rootHeader, err := rootSnapshot.Head() + require.NoError(t, err) + + // build a chain: + // G <- B1 <- B2 (resultB1) <- B3 <- B4 (resultB2, resultB3) <- B5 (sealB1) <- B6 (sealB2, sealB3) <- B7 + // test that when B4 is finalized, can only find seal for G + // when B5 is finalized, can find seal for B1 + // when B7 is finalized, can find seals for B2, B3 + + // block 1 + b1 := unittest.BlockWithParentFixture(rootHeader) + b1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b1) + require.NoError(t, err) + + // block 2(result B1) + b1Receipt := unittest.ReceiptForBlockFixture(b1) + b2 := unittest.BlockWithParentFixture(b1.Header) + b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt))) + err = state.Extend(context.Background(), b2) + require.NoError(t, err) + + // block 3 + b3 := unittest.BlockWithParentFixture(b2.Header) + b3.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b3) + require.NoError(t, err) + + // block 4 (resultB2, resultB3) + b2Receipt := unittest.ReceiptForBlockFixture(b2) + b3Receipt := unittest.ReceiptForBlockFixture(b3) + b4 := unittest.BlockWithParentFixture(b3.Header) + b4.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, + Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), b4) + require.NoError(t, err) + + // block 5 (sealB1) + b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult)) + b5 := unittest.BlockWithParentFixture(b4.Header) + b5.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b1Seal}, + }) + err = state.Extend(context.Background(), b5) + require.NoError(t, err) + + // block 6 (sealB2, sealB3) + b2Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b2Receipt.ExecutionResult)) + b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult)) + b6 := unittest.BlockWithParentFixture(b5.Header) + b6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b2Seal, b3Seal}, + }) + err = state.Extend(context.Background(), b6) + require.NoError(t, err) + + // block 7 + b7 := unittest.BlockWithParentFixture(b6.Header) + b7.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b7) + require.NoError(t, err) + + // finalizing b1 - b4 + // when B4 is finalized, can only find seal for G + err = state.Finalize(context.Background(), b1.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b2.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b3.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b4.ID()) + require.NoError(t, err) + + metrics := metrics.NewNoopCollector() + seals := bstorage.NewSeals(metrics, db) + + // can only find seal for G + _, err = seals.FinalizedSealForBlock(rootHeader.ID()) + require.NoError(t, err) + + _, err = seals.FinalizedSealForBlock(b1.ID()) + require.Error(t, err) + require.ErrorIs(t, err, storage.ErrNotFound) + + // when B5 is finalized, can find seal for B1 + err = state.Finalize(context.Background(), b5.ID()) + require.NoError(t, err) + + s1, err := seals.FinalizedSealForBlock(b1.ID()) + require.NoError(t, err) + require.Equal(t, b1Seal, s1) + + _, err = seals.FinalizedSealForBlock(b2.ID()) + require.Error(t, err) + require.ErrorIs(t, err, storage.ErrNotFound) + + // when B7 is finalized, can find seals for B2, B3 + err = state.Finalize(context.Background(), b6.ID()) + require.NoError(t, err) + + err = state.Finalize(context.Background(), b7.ID()) + require.NoError(t, err) + + s2, err := seals.FinalizedSealForBlock(b2.ID()) + require.NoError(t, err) + require.Equal(t, b2Seal, s2) + + s3, err := seals.FinalizedSealForBlock(b3.ID()) + require.NoError(t, err) + require.Equal(t, b3Seal, s3) + }) + +} + +func TestVersionBeaconIndex(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + rootHeader, err := rootSnapshot.Head() + require.NoError(t, err) + + // build a chain: + // G <- B1 <- B2 (resultB1(vb1)) <- B3 <- B4 (resultB2(vb2), resultB3(vb3)) <- B5 (sealB1) <- B6 (sealB2, sealB3) + // up until and including finalization of B5 there should be no VBs indexed + // when B5 is finalized, index VB1 + // when B6 is finalized, we can index VB2 and VB3, but (only) the last one should be indexed by seal height + + // block 1 + b1 := unittest.BlockWithParentFixture(rootHeader) + b1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b1) + require.NoError(t, err) + + vb1 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 100, + Version: "0.21.38", + }, + ), + ) + vb2 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 101, + Version: "0.21.38", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 201, + Version: "0.21.39", + }, + ), + ) + vb3 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 99, + Version: "0.21.38", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 199, + Version: "0.21.39", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 299, + Version: "0.21.40", + }, + ), + ) + + b1Receipt := unittest.ReceiptForBlockFixture(b1) + b1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb1.ServiceEvent()} + b2 := unittest.BlockWithParentFixture(b1.Header) + b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt))) + err = state.Extend(context.Background(), b2) + require.NoError(t, err) + + // block 3 + b3 := unittest.BlockWithParentFixture(b2.Header) + b3.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b3) + require.NoError(t, err) + + // block 4 (resultB2, resultB3) + b2Receipt := unittest.ReceiptForBlockFixture(b2) + b2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb2.ServiceEvent()} + + b3Receipt := unittest.ReceiptForBlockFixture(b3) + b3Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb3.ServiceEvent()} + + b4 := unittest.BlockWithParentFixture(b3.Header) + b4.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, + Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), b4) + require.NoError(t, err) + + // block 5 (sealB1) + b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult)) + b5 := unittest.BlockWithParentFixture(b4.Header) + b5.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b1Seal}, + }) + err = state.Extend(context.Background(), b5) + require.NoError(t, err) + + // block 6 (sealB2, sealB3) + b2Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b2Receipt.ExecutionResult)) + b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult)) + b6 := unittest.BlockWithParentFixture(b5.Header) + b6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b2Seal, b3Seal}, + }) + err = state.Extend(context.Background(), b6) + require.NoError(t, err) + + versionBeacons := bstorage.NewVersionBeacons(db) + + // No VB can be found before finalizing anything + vb, err := versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Nil(t, vb) + + // finalizing b1 - b5 + err = state.Finalize(context.Background(), b1.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b2.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b3.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b4.ID()) + require.NoError(t, err) + + // No VB can be found after finalizing B4 + vb, err = versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Nil(t, vb) + + // once B5 is finalized, B1 and VB1 are sealed, hence index should now find it + err = state.Finalize(context.Background(), b5.ID()) + require.NoError(t, err) + + versionBeacon, err := versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Equal(t, + &flow.SealedVersionBeacon{ + VersionBeacon: vb1, + SealHeight: b5.Header.Height, + }, + versionBeacon, + ) + + // finalizing B6 should index events sealed by B6, so VB2 and VB3 + // while we don't expect multiple VBs in one block, we index newest, so last one emitted - VB3 + err = state.Finalize(context.Background(), b6.ID()) + require.NoError(t, err) + + versionBeacon, err = versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Equal(t, + &flow.SealedVersionBeacon{ + VersionBeacon: vb3, + SealHeight: b6.Header.Height, + }, + versionBeacon, + ) + }) +} + +func TestExtendSealedBoundary(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + _, seal, err := rootSnapshot.SealedResult() + require.NoError(t, err) + finalCommit, err := state.Final().Commit() + require.NoError(t, err) + require.Equal(t, seal.FinalState, finalCommit, "original commit should be root commit") + + // Create a first block on top of the snapshot + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + + // Add a second block containing a receipt committing to the first block + block1Receipt := unittest.ReceiptForBlockFixture(block1) + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + // Add a third block containing a seal for the first block + block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{block1Seal}, + }) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + finalCommit, err = state.Final().Commit() + require.NoError(t, err) + require.Equal(t, seal.FinalState, finalCommit, "commit should not change before finalizing") + + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + finalCommit, err = state.Final().Commit() + require.NoError(t, err) + require.Equal(t, seal.FinalState, finalCommit, "commit should not change after finalizing non-sealing block") + + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + finalCommit, err = state.Final().Commit() + require.NoError(t, err) + require.Equal(t, seal.FinalState, finalCommit, "commit should not change after finalizing non-sealing block") + + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) + + finalCommit, err = state.Final().Commit() + require.NoError(t, err) + require.Equal(t, block1Seal.FinalState, finalCommit, "commit should change after finalizing sealing block") + }) +} + +func TestExtendMissingParent(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + extend := unittest.BlockFixture() + extend.Payload.Guarantees = nil + extend.Payload.Seals = nil + extend.Header.Height = 2 + extend.Header.View = 2 + extend.Header.ParentID = unittest.BlockFixture().ID() + extend.Header.PayloadHash = extend.Payload.Hash() + + err := state.Extend(context.Background(), &extend) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err), err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID)) + require.Error(t, err) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestExtendHeightTooSmall(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + extend := unittest.BlockFixture() + extend.SetPayload(flow.EmptyPayload()) + extend.Header.Height = 1 + extend.Header.View = 1 + extend.Header.ParentID = head.ID() + extend.Header.ParentView = head.View + + err = state.Extend(context.Background(), &extend) + require.NoError(t, err) + + // create another block with the same height and view, that is coming after + extend.Header.ParentID = extend.Header.ID() + extend.Header.Height = 1 + extend.Header.View = 2 + + err = state.Extend(context.Background(), &extend) + require.Error(t, err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID)) + require.Error(t, err) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestExtendHeightTooLarge(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + // set an invalid height + block.Header.Height = head.Height + 2 + + err = state.Extend(context.Background(), block) + require.Error(t, err) + }) +} + +// TestExtendInconsistentParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent +// with view of block referred by ParentID. +func TestExtendInconsistentParentView(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + // set an invalid parent view + block.Header.ParentView++ + + err = state.Extend(context.Background(), block) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err)) + }) +} + +func TestExtendBlockNotConnected(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + + // add 2 blocks, the second finalizing/sealing the state of the first + extend := unittest.BlockWithParentFixture(head) + extend.SetPayload(flow.EmptyPayload()) + + err = state.Extend(context.Background(), extend) + require.NoError(t, err) + + err = state.Finalize(context.Background(), extend.ID()) + require.NoError(t, err) + + // create a fork at view/height 1 and try to connect it to root + extend.Header.Timestamp = extend.Header.Timestamp.Add(time.Second) + extend.Header.ParentID = head.ID() + + err = state.Extend(context.Background(), extend) + require.Error(t, err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID)) + require.Error(t, err) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestExtendInvalidChainID(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + // use an invalid chain ID + block.Header.ChainID = head.ChainID + "-invalid" + + err = state.Extend(context.Background(), block) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err), err) + }) +} + +func TestExtendReceiptsNotSorted(t *testing.T) { + // TODO: this test needs to be updated: + // We don't require the receipts to be sorted by height anymore + // We could require an "parent first" ordering, which is less strict than + // a full ordering by height + unittest.SkipUnless(t, unittest.TEST_TODO, "needs update") + + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + // create block2 and block3 + block2 := unittest.BlockWithParentFixture(head) + block2.Payload.Guarantees = nil + block2.Header.PayloadHash = block2.Payload.Hash() + err := state.Extend(context.Background(), block2) + require.NoError(t, err) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.Payload.Guarantees = nil + block3.Header.PayloadHash = block3.Payload.Hash() + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + receiptA := unittest.ReceiptForBlockFixture(block3) + receiptB := unittest.ReceiptForBlockFixture(block2) + + // insert a block with payload receipts not sorted by block height. + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.Payload = &flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{receiptA.Meta(), receiptB.Meta()}, + Results: []*flow.ExecutionResult{&receiptA.ExecutionResult, &receiptB.ExecutionResult}, + } + block4.Header.PayloadHash = block4.Payload.Hash() + err = state.Extend(context.Background(), block4) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err), err) + }) +} + +func TestExtendReceiptsInvalid(t *testing.T) { + validator := mockmodule.NewReceiptValidator(t) + + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolStateAndValidator(t, rootSnapshot, validator, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + validator.On("ValidatePayload", mock.Anything).Return(nil).Once() + + // create block2 and block3 + block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + // Add a receipt for block 2 + receipt := unittest.ExecutionReceiptFixture() + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()}, + Results: []*flow.ExecutionResult{&receipt.ExecutionResult}, + }) + + // force the receipt validator to refuse this payload + validator.On("ValidatePayload", block3).Return(engine.NewInvalidInputError("")).Once() + + err = state.Extend(context.Background(), block3) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err), err) + }) +} + +func TestExtendReceiptsValid(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + + receipt3a := unittest.ReceiptForBlockFixture(block3) + receipt3b := unittest.ReceiptForBlockFixture(block3) + receipt3c := unittest.ReceiptForBlockFixture(block4) + + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{ + receipt3a.Meta(), + receipt3b.Meta(), + receipt3c.Meta(), + }, + Results: []*flow.ExecutionResult{ + &receipt3a.ExecutionResult, + &receipt3b.ExecutionResult, + &receipt3c.ExecutionResult, + }, + }) + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + }) +} + +// Tests the full flow of transitioning between epochs by finalizing a setup +// event, then a commit event, then finalizing the first block of the next epoch. +// Also tests that appropriate epoch transition events are fired. +// +// Epoch information becomes available in the protocol state in the block containing the seal +// for the block whose execution emitted the service event. +// +// ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 <- B5(R2) <- B6(S2) <- B7 <-|- B8 +// +// B3 seals B1, in which EpochSetup is emitted. +// - we can query the EpochSetup beginning with B3 +// - EpochSetupPhaseStarted triggered when B3 is finalized +// +// B6 seals B2, in which EpochCommitted is emitted. +// - we can query the EpochCommit beginning with B6 +// - EpochCommittedPhaseStarted triggered when B6 is finalized +// +// B7 is the final block of the epoch. +// B8 is the first block of the NEXT epoch. +func TestExtendEpochTransitionValid(t *testing.T) { + // create an event consumer to test epoch transition events + consumer := mockprotocol.NewConsumer(t) + consumer.On("BlockFinalized", mock.Anything) + consumer.On("BlockProcessable", mock.Anything, mock.Anything) + rootSnapshot := unittest.RootSnapshotFixture(participants) + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // set up state and mock ComplianceMetrics object + metrics := mockmodule.NewComplianceMetrics(t) + metrics.On("BlockSealed", mock.Anything) + metrics.On("SealedHeight", mock.Anything) + metrics.On("FinalizedHeight", mock.Anything) + metrics.On("BlockFinalized", mock.Anything) + + // expect epoch metric calls on bootstrap + initialCurrentEpoch := rootSnapshot.Epochs().Current() + counter, err := initialCurrentEpoch.Counter() + require.NoError(t, err) + finalView, err := initialCurrentEpoch.FinalView() + require.NoError(t, err) + initialPhase, err := rootSnapshot.Phase() + require.NoError(t, err) + metrics.On("CurrentEpochCounter", counter).Once() + metrics.On("CurrentEpochPhase", initialPhase).Once() + metrics.On("CommittedEpochFinalView", finalView).Once() + + metrics.On("CurrentEpochFinalView", finalView).Once() + + dkgPhase1FinalView, dkgPhase2FinalView, dkgPhase3FinalView, err := realprotocol.DKGPhaseViews(initialCurrentEpoch) + require.NoError(t, err) + metrics.On("CurrentDKGPhase1FinalView", dkgPhase1FinalView).Once() + metrics.On("CurrentDKGPhase2FinalView", dkgPhase2FinalView).Once() + metrics.On("CurrentDKGPhase3FinalView", dkgPhase3FinalView).Once() + + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := storeutil.StorageLayer(t, db) + protoState, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(t, err) + receiptValidator := util.MockReceiptValidator() + sealValidator := util.MockSealValidator(all.Seals) + state, err := protocol.NewFullConsensusState( + log, + tracer, + consumer, + protoState, + all.Index, + all.Payloads, + util.MockBlockTimer(), + receiptValidator, + sealValidator, + ) + require.NoError(t, err) + + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // we should begin the epoch in the staking phase + phase, err := state.AtBlockID(head.ID()).Phase() + assert.NoError(t, err) + require.Equal(t, flow.EpochPhaseStaking, phase) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical) + + // create the epoch setup event for the second epoch + epoch2Setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1FinalView+1000), + unittest.WithFirstView(epoch1FinalView+1), + ) + + // create a receipt for block 1 containing the EpochSetup event + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + seal1.ResultID = receipt1.ExecutionResult.ID() + + // add a second block with the receipt for block 1 + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + // block 3 contains the seal for block 1 + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + + // insert the block sealing the EpochSetup event + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // now that the setup event has been emitted, we should be in the setup phase + phase, err = state.AtBlockID(block3.ID()).Phase() + assert.NoError(t, err) + require.Equal(t, flow.EpochPhaseSetup, phase) + + // we should NOT be able to query epoch 2 wrt blocks before 3 + for _, blockID := range []flow.Identifier{block1.ID(), block2.ID()} { + _, err = state.AtBlockID(blockID).Epochs().Next().InitialIdentities() + require.Error(t, err) + _, err = state.AtBlockID(blockID).Epochs().Next().Clustering() + require.Error(t, err) + } + + // we should be able to query epoch 2 wrt block 3 + _, err = state.AtBlockID(block3.ID()).Epochs().Next().InitialIdentities() + assert.NoError(t, err) + _, err = state.AtBlockID(block3.ID()).Epochs().Next().Clustering() + assert.NoError(t, err) + + // only setup event is finalized, not commit, so shouldn't be able to get certain info + _, err = state.AtBlockID(block3.ID()).Epochs().Next().DKG() + require.Error(t, err) + + // insert B4 + block4 := unittest.BlockWithParentFixture(block3.Header) + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + + consumer.On("EpochSetupPhaseStarted", epoch2Setup.Counter-1, block3.Header).Once() + metrics.On("CurrentEpochPhase", flow.EpochPhaseSetup).Once() + // finalize block 3, so we can finalize subsequent blocks + // ensure an epoch phase transition when we finalize block 3 + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) + consumer.AssertCalled(t, "EpochSetupPhaseStarted", epoch2Setup.Counter-1, block3.Header) + metrics.AssertCalled(t, "CurrentEpochPhase", flow.EpochPhaseSetup) + + // now that the setup event has been emitted, we should be in the setup phase + phase, err = state.AtBlockID(block3.ID()).Phase() + require.NoError(t, err) + require.Equal(t, flow.EpochPhaseSetup, phase) + + // finalize block 4 + err = state.Finalize(context.Background(), block4.ID()) + require.NoError(t, err) + + epoch2Commit := unittest.EpochCommitFixture( + unittest.CommitWithCounter(epoch2Setup.Counter), + unittest.WithClusterQCsFromAssignments(epoch2Setup.Assignments), + unittest.WithDKGFromParticipants(epoch2Participants), + ) + + // create receipt and seal for block 2 + // the receipt for block 2 contains the EpochCommit event + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + receipt2.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Commit.ServiceEvent()} + seal2.ResultID = receipt2.ExecutionResult.ID() + + // block 5 contains the receipt for block 2 + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2))) + + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + err = state.Finalize(context.Background(), block5.ID()) + require.NoError(t, err) + + // block 6 contains the seal for block 2 + block6 := unittest.BlockWithParentFixture(block5.Header) + block6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal2}, + }) + + err = state.Extend(context.Background(), block6) + require.NoError(t, err) + + // we should NOT be able to query epoch 2 commit info wrt blocks before 6 + for _, blockID := range []flow.Identifier{block4.ID(), block5.ID()} { + _, err = state.AtBlockID(blockID).Epochs().Next().DKG() + require.Error(t, err) + } + + // now epoch 2 is fully ready, we can query anything we want about it wrt block 6 (or later) + _, err = state.AtBlockID(block6.ID()).Epochs().Next().InitialIdentities() + require.NoError(t, err) + _, err = state.AtBlockID(block6.ID()).Epochs().Next().Clustering() + require.NoError(t, err) + _, err = state.AtBlockID(block6.ID()).Epochs().Next().DKG() + assert.NoError(t, err) + + // now that the commit event has been emitted, we should be in the committed phase + phase, err = state.AtBlockID(block6.ID()).Phase() + assert.NoError(t, err) + require.Equal(t, flow.EpochPhaseCommitted, phase) + + // block 7 has the final view of the epoch, insert it, finalized after finalizing block 6 + block7 := unittest.BlockWithParentFixture(block6.Header) + block7.SetPayload(flow.EmptyPayload()) + block7.Header.View = epoch1FinalView + err = state.Extend(context.Background(), block7) + require.NoError(t, err) + + // expect epoch phase transition once we finalize block 6 + consumer.On("EpochCommittedPhaseStarted", epoch2Setup.Counter-1, block6.Header).Once() + // expect committed final view to be updated, since we are committing epoch 2 + metrics.On("CommittedEpochFinalView", epoch2Setup.FinalView).Once() + metrics.On("CurrentEpochPhase", flow.EpochPhaseCommitted).Once() + + err = state.Finalize(context.Background(), block6.ID()) + require.NoError(t, err) + + consumer.AssertCalled(t, "EpochCommittedPhaseStarted", epoch2Setup.Counter-1, block6.Header) + metrics.AssertCalled(t, "CommittedEpochFinalView", epoch2Setup.FinalView) + metrics.AssertCalled(t, "CurrentEpochPhase", flow.EpochPhaseCommitted) + + // we should still be in epoch 1 + epochCounter, err := state.AtBlockID(block4.ID()).Epochs().Current().Counter() + require.NoError(t, err) + require.Equal(t, epoch1Setup.Counter, epochCounter) + + err = state.Finalize(context.Background(), block7.ID()) + require.NoError(t, err) + + // we should still be in epoch 1, since epochs are inclusive of final view + epochCounter, err = state.AtBlockID(block7.ID()).Epochs().Current().Counter() + require.NoError(t, err) + require.Equal(t, epoch1Setup.Counter, epochCounter) + + // block 8 has a view > final view of epoch 1, it will be considered the first block of epoch 2 + block8 := unittest.BlockWithParentFixture(block7.Header) + block8.SetPayload(flow.EmptyPayload()) + // we should handle views that aren't exactly the first valid view of the epoch + block8.Header.View = epoch1FinalView + uint64(1+rand.Intn(10)) + + err = state.Extend(context.Background(), block8) + require.NoError(t, err) + + // now, at long last, we are in epoch 2 + epochCounter, err = state.AtBlockID(block8.ID()).Epochs().Current().Counter() + require.NoError(t, err) + require.Equal(t, epoch2Setup.Counter, epochCounter) + + // we should begin epoch 2 in staking phase + // how that the commit event has been emitted, we should be in the committed phase + phase, err = state.AtBlockID(block8.ID()).Phase() + assert.NoError(t, err) + require.Equal(t, flow.EpochPhaseStaking, phase) + + // expect epoch transition once we finalize block 9 + consumer.On("EpochTransition", epoch2Setup.Counter, block8.Header).Once() + metrics.On("EpochTransitionHeight", block8.Header.Height).Once() + metrics.On("CurrentEpochCounter", epoch2Setup.Counter).Once() + metrics.On("CurrentEpochPhase", flow.EpochPhaseStaking).Once() + metrics.On("CurrentEpochFinalView", epoch2Setup.FinalView).Once() + metrics.On("CurrentDKGPhase1FinalView", epoch2Setup.DKGPhase1FinalView).Once() + metrics.On("CurrentDKGPhase2FinalView", epoch2Setup.DKGPhase2FinalView).Once() + metrics.On("CurrentDKGPhase3FinalView", epoch2Setup.DKGPhase3FinalView).Once() + + // before block 9 is finalized, the epoch 1-2 boundary is unknown + _, err = state.AtBlockID(block8.ID()).Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, realprotocol.ErrEpochTransitionNotFinalized) + _, err = state.AtBlockID(block8.ID()).Epochs().Current().FirstHeight() + assert.ErrorIs(t, err, realprotocol.ErrEpochTransitionNotFinalized) + + err = state.Finalize(context.Background(), block8.ID()) + require.NoError(t, err) + + // once block 8 is finalized, epoch 2 has unambiguously begun - the epoch 1-2 boundary is known + epoch1FinalHeight, err := state.AtBlockID(block8.ID()).Epochs().Previous().FinalHeight() + require.NoError(t, err) + assert.Equal(t, block7.Header.Height, epoch1FinalHeight) + epoch2FirstHeight, err := state.AtBlockID(block8.ID()).Epochs().Current().FirstHeight() + require.NoError(t, err) + assert.Equal(t, block8.Header.Height, epoch2FirstHeight) + }) +} + +// we should be able to have conflicting forks with two different instances of +// the same service event for the same epoch +// +// /--B1<--B3(R1)<--B5(S1)<--B7 +// ROOT <--+ +// \--B2<--B4(R2)<--B6(S2)<--B8 +func TestExtendConflictingEpochEvents(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add two conflicting blocks for each service event to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + + block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + rootSetup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + + // create two conflicting epoch setup events for the next epoch (final view differs) + nextEpochSetup1 := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + nextEpochSetup2 := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+2000), // final view differs + unittest.WithFirstView(rootSetup.FinalView+1), + ) + + // add blocks containing receipts for block1 and block2 (necessary for sealing) + // block 1 receipt contains nextEpochSetup1 + block1Receipt := unittest.ReceiptForBlockFixture(block1) + block1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup1.ServiceEvent()} + + // add block 1 receipt to block 3 payload + block3 := unittest.BlockWithParentFixture(block1.Header) + block3.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // block 2 receipt contains nextEpochSetup2 + block2Receipt := unittest.ReceiptForBlockFixture(block2) + block2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup2.ServiceEvent()} + + // add block 2 receipt to block 4 payload + block4 := unittest.BlockWithParentFixture(block2.Header) + block4.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{block2Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block2Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + + // seal for block 1 + seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + + // seal for block 2 + seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult)) + + // block 5 builds on block 3, contains seal for block 1 + block5 := unittest.BlockWithParentFixture(block3.Header) + block5.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + + // block 6 builds on block 4, contains seal for block 2 + block6 := unittest.BlockWithParentFixture(block4.Header) + block6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal2}, + }) + err = state.Extend(context.Background(), block6) + require.NoError(t, err) + + // block 7 builds on block 5, contains QC for block 7 + block7 := unittest.BlockWithParentFixture(block5.Header) + err = state.Extend(context.Background(), block7) + require.NoError(t, err) + + // block 8 builds on block 6, contains QC for block 6 + block8 := unittest.BlockWithParentFixture(block6.Header) + err = state.Extend(context.Background(), block8) + require.NoError(t, err) + + // should be able query each epoch from the appropriate reference block + setup1FinalView, err := state.AtBlockID(block7.ID()).Epochs().Next().FinalView() + assert.NoError(t, err) + require.Equal(t, nextEpochSetup1.FinalView, setup1FinalView) + + setup2FinalView, err := state.AtBlockID(block8.ID()).Epochs().Next().FinalView() + assert.NoError(t, err) + require.Equal(t, nextEpochSetup2.FinalView, setup2FinalView) + }) +} + +// we should be able to have conflicting forks with two DUPLICATE instances of +// the same service event for the same epoch +// +// /--B1<--B3(R1)<--B5(S1)<--B7 +// ROOT <--+ +// \--B2<--B4(R2)<--B6(S2)<--B8 +func TestExtendDuplicateEpochEvents(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add two conflicting blocks for each service event to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + + block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + rootSetup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + + // create an epoch setup event to insert to BOTH forks + nextEpochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + + // add blocks containing receipts for block1 and block2 (necessary for sealing) + // block 1 receipt contains nextEpochSetup1 + block1Receipt := unittest.ReceiptForBlockFixture(block1) + block1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup.ServiceEvent()} + + // add block 1 receipt to block 3 payload + block3 := unittest.BlockWithParentFixture(block1.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt))) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // block 2 receipt contains nextEpochSetup2 + block2Receipt := unittest.ReceiptForBlockFixture(block2) + block2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup.ServiceEvent()} + + // add block 2 receipt to block 4 payload + block4 := unittest.BlockWithParentFixture(block2.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block2Receipt))) + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + + // seal for block 1 + seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + + // seal for block 2 + seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult)) + + // block 5 builds on block 3, contains seal for block 1 + block5 := unittest.BlockWithParentFixture(block3.Header) + block5.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + + // block 6 builds on block 4, contains seal for block 2 + block6 := unittest.BlockWithParentFixture(block4.Header) + block6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal2}, + }) + err = state.Extend(context.Background(), block6) + require.NoError(t, err) + + // block 7 builds on block 5, contains QC for block 7 + block7 := unittest.BlockWithParentFixture(block5.Header) + err = state.Extend(context.Background(), block7) + require.NoError(t, err) + + // block 8 builds on block 6, contains QC for block 6 + // at this point we are inserting the duplicate EpochSetup, should not error + block8 := unittest.BlockWithParentFixture(block6.Header) + err = state.Extend(context.Background(), block8) + require.NoError(t, err) + + // should be able query each epoch from the appropriate reference block + finalView, err := state.AtBlockID(block7.ID()).Epochs().Next().FinalView() + assert.NoError(t, err) + require.Equal(t, nextEpochSetup.FinalView, finalView) + + finalView, err = state.AtBlockID(block8.ID()).Epochs().Next().FinalView() + assert.NoError(t, err) + require.Equal(t, nextEpochSetup.FinalView, finalView) + }) +} + +// TestExtendEpochSetupInvalid tests that incorporating an invalid EpochSetup +// service event should trigger epoch fallback when the fork is finalized. +func TestExtendEpochSetupInvalid(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + + // setupState initializes the protocol state for a test case + // * creates and finalizes a new block for the first seal to reference + // * creates a factory method for test cases to generated valid EpochSetup events + setupState := func(t *testing.T, db *badger.DB, state *protocol.ParticipantState) ( + *flow.Block, + func(...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal), + ) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + unittest.InsertAndFinalize(t, state, block1) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical) + + // this function will return a VALID setup event and seal, we will modify + // in different ways in each test case + createSetupEvent := func(opts ...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { + setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1Setup.FinalView+1000), + unittest.WithFirstView(epoch1Setup.FinalView+1), + ) + for _, apply := range opts { + apply(setup) + } + receipt, seal := unittest.ReceiptAndSealForBlock(block1) + receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent()} + seal.ResultID = receipt.ExecutionResult.ID() + return setup, receipt, seal + } + + return block1, createSetupEvent + } + + // expect a setup event with wrong counter to trigger EECC without error + t.Run("wrong counter (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup := setupState(t, db, state) + + _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { + setup.Counter = rand.Uint64() + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) + + // expect a setup event with wrong final view to trigger EECC without error + t.Run("invalid final view (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup := setupState(t, db, state) + + _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { + setup.FinalView = block1.Header.View + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) + + // expect a setup event with empty seed to trigger EECC without error + t.Run("empty seed (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup := setupState(t, db, state) + + _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { + setup.RandomSource = nil + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) +} + +// TestExtendEpochCommitInvalid tests that incorporating an invalid EpochCommit +// service event should trigger epoch fallback when the fork is finalized. +func TestExtendEpochCommitInvalid(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + + // setupState initializes the protocol state for a test case + // * creates and finalizes a new block for the first seal to reference + // * creates a factory method for test cases to generated valid EpochSetup events + // * creates a factory method for test cases to generated valid EpochCommit events + setupState := func(t *testing.T, state *protocol.ParticipantState) ( + *flow.Block, + func(*flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal), + func(*flow.Block, ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal), + ) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + unittest.InsertAndFinalize(t, state, block1) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + + // swap consensus node for a new one for epoch 2 + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) + epoch2Participants := append( + participants.Filter(filter.Not(filter.HasRole(flow.RoleConsensus))), + epoch2NewParticipant, + ).Sort(flow.Canonical) + + // factory method to create a valid EpochSetup method w.r.t. the generated state + createSetup := func(block *flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { + setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1Setup.FinalView+1000), + unittest.WithFirstView(epoch1Setup.FinalView+1), + ) + + receipt, seal := unittest.ReceiptAndSealForBlock(block) + receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent()} + seal.ResultID = receipt.ExecutionResult.ID() + return setup, receipt, seal + } + + // factory method to create a valid EpochCommit method w.r.t. the generated state + createCommit := func(block *flow.Block, opts ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal) { + commit := unittest.EpochCommitFixture( + unittest.CommitWithCounter(epoch1Setup.Counter+1), + unittest.WithDKGFromParticipants(epoch2Participants), + ) + for _, apply := range opts { + apply(commit) + } + receipt, seal := unittest.ReceiptAndSealForBlock(block) + receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{commit.ServiceEvent()} + seal.ResultID = receipt.ExecutionResult.ID() + return commit, receipt, seal + } + + return block1, createSetup, createCommit + } + + t.Run("without setup (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, _, createCommit := setupState(t, state) + + _, receipt, seal := createCommit(block1) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) + + // expect a commit event with wrong counter to trigger EECC without error + t.Run("inconsistent counter (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup, createCommit := setupState(t, state) + + // seal block 1, in which EpochSetup was emitted + epoch2Setup, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) + + _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { + commit.Counter = epoch2Setup.Counter + 1 + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) + + // expect a commit event with wrong cluster QCs to trigger EECC without error + t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup, createCommit := setupState(t, state) + + // seal block 1, in which EpochSetup was emitted + _, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) + + _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { + commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(unittest.QuorumCertificateWithSignerIDsFixture())) + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) + + // expect a commit event with wrong dkg participants to trigger EECC without error + t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block1, createSetup, createCommit := setupState(t, state) + + // seal block 1, in which EpochSetup was emitted + _, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) + + _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { + // add an extra dkg key + commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) + }) + + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) + }) + }) +} + +// if we reach the first block of the next epoch before both setup and commit +// service events are finalized, the chain should halt +// +// ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 +func TestExtendEpochTransitionWithoutCommit(t *testing.T) { + + // skipping because this case will now result in emergency epoch continuation kicking in + unittest.SkipUnless(t, unittest.TEST_TODO, "disabled as the current implementation uses a temporary fallback measure in this case (triggers EECC), rather than returning an error") + + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical) + + // create the epoch setup event for the second epoch + epoch2Setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1FinalView+1000), + unittest.WithFirstView(epoch1FinalView+1), + ) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + + // add a block containing a receipt for block 1 + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + // block 3 seals block 1 + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // block 4 will be the first block for epoch 2 + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.Header.View = epoch1Setup.FinalView + 1 + + err = state.Extend(context.Background(), block4) + require.Error(t, err) + }) +} + +// TestEmergencyEpochFallback tests that epoch emergency fallback is triggered +// when an epoch fails to be committed before the epoch commitment deadline, +// or when an invalid service event (indicating service account smart contract bug) +// is sealed. +func TestEmergencyEpochFallback(t *testing.T) { + + // if we finalize the first block past the epoch commitment deadline while + // in the EpochStaking phase, EECC should be triggered + // + // Epoch Commitment Deadline + // | Epoch Boundary + // | | + // v v + // ROOT <- B1 <- B2 + t.Run("passed epoch commitment deadline in EpochStaking phase - should trigger EECC", func(t *testing.T) { + + rootSnapshot := unittest.RootSnapshotFixture(participants) + metricsMock := mockmodule.NewComplianceMetrics(t) + mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) + + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold() + require.NoError(t, err) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold + + // finalizing block 1 should trigger EECC + metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() + + // we begin the epoch in the EpochStaking phase and + // block 1 will be the first block on or past the epoch commitment deadline + block1 := unittest.BlockWithParentFixture(head) + block1.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization + + // block 2 will be the first block past the first epoch boundary + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.Header.View = epoch1FinalView + 1 + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + // since EECC has been triggered, epoch transition metrics should not be updated + metricsMock.AssertNotCalled(t, "EpochTransition", mock.Anything, mock.Anything) + metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch1Setup.Counter+1) + }) + }) + + // if we finalize the first block past the epoch commitment deadline while + // in the EpochSetup phase, EECC should be triggered + // + // Epoch Commitment Deadline + // | Epoch Boundary + // | | + // v v + // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 + t.Run("passed epoch commitment deadline in EpochSetup phase - should trigger EECC", func(t *testing.T) { + + rootSnapshot := unittest.RootSnapshotFixture(participants) + metricsMock := mockmodule.NewComplianceMetrics(t) + mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) + + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold() + require.NoError(t, err) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical) + + // create the epoch setup event for the second epoch + epoch2Setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1FinalView+1000), + unittest.WithFirstView(epoch1FinalView+1), + ) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + seal1.ResultID = receipt1.ExecutionResult.ID() + + // add a block containing a receipt for block 1 + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + // block 3 seals block 1 and will be the first block on or past the epoch commitment deadline + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // finalizing block 3 should trigger EECC + metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() + + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization + + // block 4 will be the first block past the first epoch boundary + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.Header.View = epoch1FinalView + 1 + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + err = state.Finalize(context.Background(), block4.ID()) + require.NoError(t, err) + + // since EECC has been triggered, epoch transition metrics should not be updated + metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything) + metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) + }) + }) + + // if an invalid epoch service event is incorporated, we should: + // - not apply the phase transition corresponding to the invalid service event + // - immediately trigger EECC + // + // Epoch Boundary + // | + // v + // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 + t.Run("epoch transition with invalid service event - should trigger EECC", func(t *testing.T) { + + rootSnapshot := unittest.RootSnapshotFixture(participants) + metricsMock := mockmodule.NewComplianceMetrics(t) + mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) + + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + // add a block for the first seal to reference + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical) + + // create the epoch setup event for the second epoch + // this event is invalid because it used a non-contiguous first view + epoch2Setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1FinalView+1000), + unittest.WithFirstView(epoch1FinalView+10), // invalid first view + ) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + seal1.ResultID = receipt1.ExecutionResult.ID() + + // add a block containing a receipt for block 1 + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + // block 3 is where the service event state change comes into effect + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + err = state.Extend(context.Background(), block3) + require.NoError(t, err) + + // incorporating the service event should trigger EECC + metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() + + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization + + // block 5 is the first block past the current epoch boundary + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.Header.View = epoch1Setup.FinalView + 1 + err = state.Extend(context.Background(), block4) + require.NoError(t, err) + err = state.Finalize(context.Background(), block4.ID()) + require.NoError(t, err) + + // since EECC has been triggered, epoch transition metrics should not be updated + metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything) + metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) + }) + }) +} + +func TestExtendInvalidSealsInBlock(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := storeutil.StorageLayer(t, db) + + // create a event consumer to test epoch transition events + distributor := events.NewDistributor() + consumer := mockprotocol.NewConsumer(t) + distributor.AddConsumer(consumer) + consumer.On("BlockProcessable", mock.Anything, mock.Anything) + + rootSnapshot := unittest.RootSnapshotFixture(participants) + + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(t, err) + + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block1 := unittest.BlockWithParentFixture(head) + block1.Payload.Guarantees = nil + block1.Header.PayloadHash = block1.Payload.Hash() + + block1Receipt := unittest.ReceiptForBlockFixture(block1) + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt))) + + block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{block1Seal}, + }) + + sealValidator := mockmodule.NewSealValidator(t) + sealValidator.On("Validate", mock.Anything). + Return(func(candidate *flow.Block) *flow.Seal { + if candidate.ID() == block3.ID() { + return nil + } + seal, _ := all.Seals.HighestInFork(candidate.Header.ParentID) + return seal + }, func(candidate *flow.Block) error { + if candidate.ID() == block3.ID() { + return engine.NewInvalidInputError("") + } + _, err := all.Seals.HighestInFork(candidate.Header.ParentID) + return err + }). + Times(3) + + fullState, err := protocol.NewFullConsensusState( + log, + tracer, + consumer, + state, + all.Index, + all.Payloads, + util.MockBlockTimer(), + util.MockReceiptValidator(), + sealValidator, + ) + require.NoError(t, err) + + err = fullState.Extend(context.Background(), block1) + require.NoError(t, err) + err = fullState.Extend(context.Background(), block2) + require.NoError(t, err) + err = fullState.Extend(context.Background(), block3) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err)) + }) +} + +func TestHeaderExtendValid(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + _, seal, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + extend := unittest.BlockWithParentFixture(head) + extend.SetPayload(flow.EmptyPayload()) + + err = state.ExtendCertified(context.Background(), extend, unittest.CertifyBlock(extend.Header)) + require.NoError(t, err) + + finalCommit, err := state.Final().Commit() + require.NoError(t, err) + require.Equal(t, seal.FinalState, finalCommit) + }) +} + +func TestHeaderExtendMissingParent(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + extend := unittest.BlockFixture() + extend.Payload.Guarantees = nil + extend.Payload.Seals = nil + extend.Header.Height = 2 + extend.Header.View = 2 + extend.Header.ParentID = unittest.BlockFixture().ID() + extend.Header.PayloadHash = extend.Payload.Hash() + + err := state.ExtendCertified(context.Background(), &extend, unittest.CertifyBlock(extend.Header)) + require.Error(t, err) + require.False(t, st.IsInvalidExtensionError(err), err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID)) + require.Error(t, err) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestHeaderExtendHeightTooSmall(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block1 := unittest.BlockWithParentFixture(head) + + // create another block that points to the previous block `extend` as parent + // but has _same_ height as parent. This violates the condition that a child's + // height must increment the parent's height by one, i.e. it should be rejected + // by the follower right away + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.Header.Height = block1.Header.Height + + err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block2, unittest.CertifyBlock(block2.Header)) + require.False(t, st.IsInvalidExtensionError(err)) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID)) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestHeaderExtendHeightTooLarge(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + // set an invalid height + block.Header.Height = head.Height + 2 + + err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + require.False(t, st.IsInvalidExtensionError(err)) + }) +} + +// TestExtendBlockProcessable tests that BlockProcessable is called correctly and doesn't produce duplicates of same notifications +// when extending blocks with and without certifying QCs. +func TestExtendBlockProcessable(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + consumer := mockprotocol.NewConsumer(t) + util.RunWithFullProtocolStateAndConsumer(t, rootSnapshot, consumer, func(db *badger.DB, state *protocol.ParticipantState) { + block := unittest.BlockWithParentFixture(head) + child := unittest.BlockWithParentFixture(block.Header) + grandChild := unittest.BlockWithParentFixture(child.Header) + + // extend block using certifying QC, expect that BlockProcessable will be emitted once + consumer.On("BlockProcessable", block.Header, child.Header.QuorumCertificate()).Once() + err := state.ExtendCertified(context.Background(), block, child.Header.QuorumCertificate()) + require.NoError(t, err) + + // extend block without certifying QC, expect that BlockProcessable won't be called + err = state.Extend(context.Background(), child) + require.NoError(t, err) + consumer.AssertNumberOfCalls(t, "BlockProcessable", 1) + + // extend block using certifying QC, expect that BlockProcessable will be emitted twice. + // One for parent block and second for current block. + grandChildCertifyingQC := unittest.CertifyBlock(grandChild.Header) + consumer.On("BlockProcessable", child.Header, grandChild.Header.QuorumCertificate()).Once() + consumer.On("BlockProcessable", grandChild.Header, grandChildCertifyingQC).Once() + err = state.ExtendCertified(context.Background(), grandChild, grandChildCertifyingQC) + require.NoError(t, err) + }) +} + +// TestFollowerHeaderExtendBlockNotConnected tests adding an orphaned block to the follower state. +// Specifically, we add 2 blocks, where: +// first block is added and then finalized; +// second block is a sibling to the finalized block +// The Follower should accept this block since tracking of orphan blocks is implemented by another component. +func TestFollowerHeaderExtendBlockNotConnected(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block1 := unittest.BlockWithParentFixture(head) + err = state.ExtendCertified(context.Background(), block1, unittest.CertifyBlock(block1.Header)) + require.NoError(t, err) + + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + // create a fork at view/height 1 and try to connect it to root + block2 := unittest.BlockWithParentFixture(head) + err = state.ExtendCertified(context.Background(), block2, unittest.CertifyBlock(block2.Header)) + require.NoError(t, err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID)) + require.NoError(t, err) + }) +} + +// TestParticipantHeaderExtendBlockNotConnected tests adding an orphaned block to the consensus participant state. +// Specifically, we add 2 blocks, where: +// first block is added and then finalized; +// second block is a sibling to the finalized block +// The Participant should reject this block as an outdated chain extension +func TestParticipantHeaderExtendBlockNotConnected(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block1 := unittest.BlockWithParentFixture(head) + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + // create a fork at view/height 1 and try to connect it to root + block2 := unittest.BlockWithParentFixture(head) + err = state.Extend(context.Background(), block2) + require.True(t, st.IsOutdatedExtensionError(err), err) + + // verify seal not indexed + var sealID flow.Identifier + err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID)) + require.ErrorIs(t, err, stoerr.ErrNotFound) + }) +} + +func TestHeaderExtendHighestSeal(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + // create block2 and block3 + block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(flow.EmptyPayload()) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.EmptyPayload()) + + err := state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate()) + require.NoError(t, err) + + // create receipts and seals for block2 and block3 + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) + + // include the seals in block4 + block4 := unittest.BlockWithParentFixture(block3.Header) + // include receipts and results + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3, receipt2))) + + // include the seals in block4 + block5 := unittest.BlockWithParentFixture(block4.Header) + // placing seals in the reversed order to test + // Extend will pick the highest sealed block + block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal3, seal2))) + + err = state.ExtendCertified(context.Background(), block3, block4.Header.QuorumCertificate()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block4, block5.Header.QuorumCertificate()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block5, unittest.CertifyBlock(block5.Header)) + require.NoError(t, err) + + finalCommit, err := state.AtBlockID(block5.ID()).Commit() + require.NoError(t, err) + require.Equal(t, seal3.FinalState, finalCommit) + }) +} + +// TestExtendCertifiedInvalidQC checks if ExtendCertified performs a sanity check of certifying QC. +func TestExtendCertifiedInvalidQC(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + // create child block + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + + t.Run("qc-invalid-view", func(t *testing.T) { + certifyingQC := unittest.CertifyBlock(block.Header) + certifyingQC.View++ // invalidate block view + err = state.ExtendCertified(context.Background(), block, certifyingQC) + require.Error(t, err) + require.False(t, st.IsOutdatedExtensionError(err)) + }) + t.Run("qc-invalid-block-id", func(t *testing.T) { + certifyingQC := unittest.CertifyBlock(block.Header) + certifyingQC.BlockID = unittest.IdentifierFixture() // invalidate blockID + err = state.ExtendCertified(context.Background(), block, certifyingQC) + require.Error(t, err) + require.False(t, st.IsOutdatedExtensionError(err)) + }) + }) +} + +// TestExtendInvalidGuarantee checks if Extend method will reject invalid blocks that contain +// guarantees with invalid guarantors +func TestExtendInvalidGuarantee(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + // create a valid block + head, err := rootSnapshot.Head() + require.NoError(t, err) + + cluster, err := unittest.SnapshotClusterByIndex(rootSnapshot, 0) + require.NoError(t, err) + + // prepare for a valid guarantor signer indices to be used in the valid block + all := cluster.Members().NodeIDs() + validSignerIndices, err := signature.EncodeSignersToIndices(all, all) + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + payload := flow.EmptyPayload() + payload.Guarantees = []*flow.CollectionGuarantee{ + { + ChainID: cluster.ChainID(), + ReferenceBlockID: head.ID(), + SignerIndices: validSignerIndices, + }, + } + + // now the valid block has a guarantee in the payload with valid signer indices. + block.SetPayload(payload) + + // check Extend should accept this valid block + err = state.Extend(context.Background(), block) + require.NoError(t, err) + + // now the guarantee has invalid signer indices: the checksum should have 4 bytes, but it only has 1 + payload.Guarantees[0].SignerIndices = []byte{byte(1)} + + // create new block that has invalid collection guarantee + block = unittest.BlockWithParentFixture(head) + block.SetPayload(payload) + + err = state.Extend(context.Background(), block) + require.True(t, signature.IsInvalidSignerIndicesError(err), err) + require.ErrorIs(t, err, signature.ErrInvalidChecksum) + require.True(t, st.IsInvalidExtensionError(err), err) + + // now the guarantee has invalid signer indices: the checksum should have 4 bytes, but it only has 1 + checksumMismatch := make([]byte, len(validSignerIndices)) + copy(checksumMismatch, validSignerIndices) + checksumMismatch[0] = byte(1) + if checksumMismatch[0] == validSignerIndices[0] { + checksumMismatch[0] = byte(2) + } + payload.Guarantees[0].SignerIndices = checksumMismatch + err = state.Extend(context.Background(), block) + require.True(t, signature.IsInvalidSignerIndicesError(err), err) + require.ErrorIs(t, err, signature.ErrInvalidChecksum) + require.True(t, st.IsInvalidExtensionError(err), err) + + // let's test even if the checksum is correct, but signer indices is still wrong because the tailing are not 0, + // then the block should still be rejected. + wrongTailing := make([]byte, len(validSignerIndices)) + copy(wrongTailing, validSignerIndices) + wrongTailing[len(wrongTailing)-1] = byte(255) + + payload.Guarantees[0].SignerIndices = wrongTailing + err = state.Extend(context.Background(), block) + require.Error(t, err) + require.True(t, signature.IsInvalidSignerIndicesError(err), err) + require.ErrorIs(t, err, signature.ErrIllegallyPaddedBitVector) + require.True(t, st.IsInvalidExtensionError(err), err) + + // test imcompatible bit vector length + wrongbitVectorLength := validSignerIndices[0 : len(validSignerIndices)-1] + payload.Guarantees[0].SignerIndices = wrongbitVectorLength + err = state.Extend(context.Background(), block) + require.True(t, signature.IsInvalidSignerIndicesError(err), err) + require.ErrorIs(t, err, signature.ErrIncompatibleBitVectorLength) + require.True(t, st.IsInvalidExtensionError(err), err) + + // revert back to good value + payload.Guarantees[0].SignerIndices = validSignerIndices + + // test the ReferenceBlockID is not found + payload.Guarantees[0].ReferenceBlockID = flow.ZeroID + err = state.Extend(context.Background(), block) + require.ErrorIs(t, err, storage.ErrNotFound) + require.True(t, st.IsInvalidExtensionError(err), err) + + // revert back to good value + payload.Guarantees[0].ReferenceBlockID = head.ID() + + // TODO: test the guarantee has bad reference block ID that would return protocol.ErrNextEpochNotCommitted + // this case is not easy to create, since the test case has no such block yet. + // we need to refactor the ParticipantState to add a guaranteeValidator, so that we can mock it and + // return the protocol.ErrNextEpochNotCommitted for testing + + // test the guarantee has wrong chain ID, and should return ErrClusterNotFound + payload.Guarantees[0].ChainID = flow.ChainID("some_bad_chain_ID") + err = state.Extend(context.Background(), block) + require.Error(t, err) + require.ErrorIs(t, err, realprotocol.ErrClusterNotFound) + require.True(t, st.IsInvalidExtensionError(err), err) + }) +} + +// If block B is finalized and contains a seal for block A, then A is the last sealed block +func TestSealed(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + // block 1 will be sealed + block1 := unittest.BlockWithParentFixture(head) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + // block 2 contains receipt for block 1 + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + + err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate()) + require.NoError(t, err) + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + + // block 3 contains seal for block 1 + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(flow.Payload{ + Seals: []*flow.Seal{seal1}, + }) + + err = state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate()) + require.NoError(t, err) + err = state.Finalize(context.Background(), block2.ID()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block3, unittest.CertifyBlock(block3.Header)) + require.NoError(t, err) + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) + + sealed, err := state.Sealed().Head() + require.NoError(t, err) + require.Equal(t, block1.ID(), sealed.ID()) + }) +} + +// Test that when adding a block to database, there are only two cases at any point of time: +// 1) neither the block header, nor the payload index exist in database +// 2) both the block header and the payload index can be found in database +// A non atomic bug would be: header is found in DB, but payload index is not found +func TestCacheAtomicity(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFollowerProtocolStateAndHeaders(t, rootSnapshot, + func(db *badger.DB, state *protocol.FollowerState, headers storage.Headers, index storage.Index) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + blockID := block.ID() + + // check 100 times to see if either 1) or 2) satisfies + var wg sync.WaitGroup + wg.Add(1) + go func(blockID flow.Identifier) { + for i := 0; i < 100; i++ { + _, err := headers.ByBlockID(blockID) + if errors.Is(err, stoerr.ErrNotFound) { + continue + } + require.NoError(t, err) + + _, err = index.ByBlockID(blockID) + require.NoError(t, err, "found block ID, but index is missing, DB updates is non-atomic") + } + wg.Done() + }(blockID) + + // storing the block to database, which supposed to be atomic updates to headers and index, + // both to badger database and the cache. + err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + require.NoError(t, err) + wg.Wait() + }) +} + +// TestHeaderInvalidTimestamp tests that extending header with invalid timestamp results in sentinel error +func TestHeaderInvalidTimestamp(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + all := storeutil.StorageLayer(t, db) + + // create a event consumer to test epoch transition events + distributor := events.NewDistributor() + consumer := mockprotocol.NewConsumer(t) + distributor.AddConsumer(consumer) + + block, result, seal := unittest.BootstrapFixture(participants) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID())) + rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) + require.NoError(t, err) + + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(t, err) + + blockTimer := &mockprotocol.BlockTimer{} + blockTimer.On("Validate", mock.Anything, mock.Anything).Return(realprotocol.NewInvalidBlockTimestamp("")) + + fullState, err := protocol.NewFullConsensusState( + log, + tracer, + consumer, + state, + all.Index, + all.Payloads, + blockTimer, + util.MockReceiptValidator(), + util.MockSealValidator(all.Seals), + ) + require.NoError(t, err) + + extend := unittest.BlockWithParentFixture(block.Header) + extend.Payload.Guarantees = nil + extend.Header.PayloadHash = extend.Payload.Hash() + + err = fullState.Extend(context.Background(), extend) + assert.Error(t, err, "a proposal with invalid timestamp has to be rejected") + assert.True(t, st.IsInvalidExtensionError(err), "if timestamp is invalid it should return invalid block error") + }) +} + +// TestProtocolStateIdempotent tests that both participant and follower states correctly process adding same block twice +// where second extend doesn't result in an error and effectively is no-op. +func TestProtocolStateIdempotent(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + t.Run("follower", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { + block := unittest.BlockWithParentFixture(head) + err := state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + require.NoError(t, err) + + // same operation should be no-op + err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + require.NoError(t, err) + }) + }) + t.Run("participant", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + block := unittest.BlockWithParentFixture(head) + err := state.Extend(context.Background(), block) + require.NoError(t, err) + + // same operation should be no-op + err = state.Extend(context.Background(), block) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) + require.NoError(t, err) + }) + }) +} + +func assertEpochEmergencyFallbackTriggered(t *testing.T, state realprotocol.State, expected bool) { + triggered, err := state.Params().EpochFallbackTriggered() + require.NoError(t, err) + assert.Equal(t, expected, triggered) +} + +// mockMetricsForRootSnapshot mocks the given metrics mock object to expect all +// metrics which are set during bootstrapping and building blocks. +func mockMetricsForRootSnapshot(metricsMock *mockmodule.ComplianceMetrics, rootSnapshot *inmem.Snapshot) { + metricsMock.On("CurrentEpochCounter", rootSnapshot.Encodable().Epochs.Current.Counter) + metricsMock.On("CurrentEpochPhase", rootSnapshot.Encodable().Phase) + metricsMock.On("CurrentEpochFinalView", rootSnapshot.Encodable().Epochs.Current.FinalView) + metricsMock.On("CommittedEpochFinalView", rootSnapshot.Encodable().Epochs.Current.FinalView) + metricsMock.On("CurrentDKGPhase1FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase1FinalView) + metricsMock.On("CurrentDKGPhase2FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase2FinalView) + metricsMock.On("CurrentDKGPhase3FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase3FinalView) + metricsMock.On("BlockSealed", mock.Anything) + metricsMock.On("BlockFinalized", mock.Anything) + metricsMock.On("FinalizedHeight", mock.Anything) + metricsMock.On("SealedHeight", mock.Anything) +} diff --git a/state/protocol/pebble/params.go b/state/protocol/pebble/params.go new file mode 100644 index 00000000000..52a447f7351 --- /dev/null +++ b/state/protocol/pebble/params.go @@ -0,0 +1,131 @@ +package badger + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage/badger/operation" +) + +type Params struct { + state *State +} + +var _ protocol.Params = (*Params)(nil) + +func (p Params) ChainID() (flow.ChainID, error) { + + // retrieve root header + root, err := p.FinalizedRoot() + if err != nil { + return "", fmt.Errorf("could not get root: %w", err) + } + + return root.ChainID, nil +} + +func (p Params) SporkID() (flow.Identifier, error) { + + var sporkID flow.Identifier + err := p.state.db.View(operation.RetrieveSporkID(&sporkID)) + if err != nil { + return flow.ZeroID, fmt.Errorf("could not get spork id: %w", err) + } + + return sporkID, nil +} + +func (p Params) SporkRootBlockHeight() (uint64, error) { + var sporkRootBlockHeight uint64 + err := p.state.db.View(operation.RetrieveSporkRootBlockHeight(&sporkRootBlockHeight)) + if err != nil { + return 0, fmt.Errorf("could not get spork root block height: %w", err) + } + + return sporkRootBlockHeight, nil +} + +func (p Params) ProtocolVersion() (uint, error) { + + var version uint + err := p.state.db.View(operation.RetrieveProtocolVersion(&version)) + if err != nil { + return 0, fmt.Errorf("could not get protocol version: %w", err) + } + + return version, nil +} + +func (p Params) EpochCommitSafetyThreshold() (uint64, error) { + + var threshold uint64 + err := p.state.db.View(operation.RetrieveEpochCommitSafetyThreshold(&threshold)) + if err != nil { + return 0, fmt.Errorf("could not get epoch commit safety threshold") + } + return threshold, nil +} + +func (p Params) EpochFallbackTriggered() (bool, error) { + var triggered bool + err := p.state.db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) + if err != nil { + return false, fmt.Errorf("could not check epoch fallback triggered: %w", err) + } + return triggered, nil +} + +func (p Params) FinalizedRoot() (*flow.Header, error) { + + // look up root block ID + var rootID flow.Identifier + err := p.state.db.View(operation.LookupBlockHeight(p.state.finalizedRootHeight, &rootID)) + if err != nil { + return nil, fmt.Errorf("could not look up root header: %w", err) + } + + // retrieve root header + header, err := p.state.headers.ByBlockID(rootID) + if err != nil { + return nil, fmt.Errorf("could not retrieve root header: %w", err) + } + + return header, nil +} + +func (p Params) SealedRoot() (*flow.Header, error) { + // look up root block ID + var rootID flow.Identifier + err := p.state.db.View(operation.LookupBlockHeight(p.state.sealedRootHeight, &rootID)) + + if err != nil { + return nil, fmt.Errorf("could not look up root header: %w", err) + } + + // retrieve root header + header, err := p.state.headers.ByBlockID(rootID) + if err != nil { + return nil, fmt.Errorf("could not retrieve root header: %w", err) + } + + return header, nil +} + +func (p Params) Seal() (*flow.Seal, error) { + + // look up root header + var rootID flow.Identifier + err := p.state.db.View(operation.LookupBlockHeight(p.state.finalizedRootHeight, &rootID)) + if err != nil { + return nil, fmt.Errorf("could not look up root header: %w", err) + } + + // retrieve the root seal + seal, err := p.state.seals.HighestInFork(rootID) + if err != nil { + return nil, fmt.Errorf("could not retrieve root seal: %w", err) + } + + return seal, nil +} diff --git a/state/protocol/pebble/snapshot.go b/state/protocol/pebble/snapshot.go new file mode 100644 index 00000000000..6dbba18b09f --- /dev/null +++ b/state/protocol/pebble/snapshot.go @@ -0,0 +1,578 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/model/flow/mapfunc" + "github.com/onflow/flow-go/state/fork" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/invalid" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" +) + +// Snapshot implements the protocol.Snapshot interface. +// It represents a read-only immutable snapshot of the protocol state at the +// block it is constructed with. It allows efficient access to data associated directly +// with blocks at a given state (finalized, sealed), such as the related header, commit, +// seed or descending blocks. A block snapshot can lazily convert to an epoch snapshot in +// order to make data associated directly with epochs accessible through its API. +type Snapshot struct { + state *State + blockID flow.Identifier // reference block for this snapshot +} + +// FinalizedSnapshot represents a read-only immutable snapshot of the protocol state +// at a finalized block. It is guaranteed to have a header available. +type FinalizedSnapshot struct { + Snapshot + header *flow.Header +} + +var _ protocol.Snapshot = (*Snapshot)(nil) +var _ protocol.Snapshot = (*FinalizedSnapshot)(nil) + +// newSnapshotWithIncorporatedReferenceBlock creates a new state snapshot with the given reference block. +// CAUTION: The caller is responsible for ensuring that the reference block has been incorporated. +func newSnapshotWithIncorporatedReferenceBlock(state *State, blockID flow.Identifier) *Snapshot { + return &Snapshot{ + state: state, + blockID: blockID, + } +} + +// NewFinalizedSnapshot instantiates a `FinalizedSnapshot`. +// CAUTION: the header's ID _must_ match `blockID` (not checked) +func NewFinalizedSnapshot(state *State, blockID flow.Identifier, header *flow.Header) *FinalizedSnapshot { + return &FinalizedSnapshot{ + Snapshot: Snapshot{ + state: state, + blockID: blockID, + }, + header: header, + } +} + +func (s *FinalizedSnapshot) Head() (*flow.Header, error) { + return s.header, nil +} + +func (s *Snapshot) Head() (*flow.Header, error) { + head, err := s.state.headers.ByBlockID(s.blockID) + return head, err +} + +// QuorumCertificate (QC) returns a valid quorum certificate pointing to the +// header at this snapshot. +// The sentinel error storage.ErrNotFound is returned if the QC is unknown. +func (s *Snapshot) QuorumCertificate() (*flow.QuorumCertificate, error) { + qc, err := s.state.qcs.ByBlockID(s.blockID) + if err != nil { + return nil, fmt.Errorf("could not retrieve quorum certificate for (%x): %w", s.blockID, err) + } + return qc, nil +} + +func (s *Snapshot) Phase() (flow.EpochPhase, error) { + status, err := s.state.epoch.statuses.ByBlockID(s.blockID) + if err != nil { + return flow.EpochPhaseUndefined, fmt.Errorf("could not retrieve epoch status: %w", err) + } + phase, err := status.Phase() + return phase, err +} + +func (s *Snapshot) Identities(selector flow.IdentityFilter) (flow.IdentityList, error) { + + // TODO: CAUTION SHORTCUT + // we retrieve identities based on the initial identity table from the EpochSetup + // event here -- this will need revision to support mid-epoch identity changes + // once slashing is implemented + + status, err := s.state.epoch.statuses.ByBlockID(s.blockID) + if err != nil { + return nil, err + } + + setup, err := s.state.epoch.setups.ByID(status.CurrentEpoch.SetupID) + if err != nil { + return nil, err + } + + // sort the identities so the 'IsCached' binary search works + identities := setup.Participants.Sort(flow.Canonical) + + // get identities that are in either last/next epoch but NOT in the current epoch + var otherEpochIdentities flow.IdentityList + phase, err := status.Phase() + if err != nil { + return nil, fmt.Errorf("could not get phase: %w", err) + } + switch phase { + // during staking phase (the beginning of the epoch) we include identities + // from the previous epoch that are now un-staking + case flow.EpochPhaseStaking: + + if !status.HasPrevious() { + break + } + + previousSetup, err := s.state.epoch.setups.ByID(status.PreviousEpoch.SetupID) + if err != nil { + return nil, fmt.Errorf("could not get previous epoch setup event: %w", err) + } + + for _, identity := range previousSetup.Participants { + exists := identities.Exists(identity) + // add identity from previous epoch that is not in current epoch + if !exists { + otherEpochIdentities = append(otherEpochIdentities, identity) + } + } + + // during setup and committed phases (the end of the epoch) we include + // identities that will join in the next epoch + case flow.EpochPhaseSetup, flow.EpochPhaseCommitted: + + nextSetup, err := s.state.epoch.setups.ByID(status.NextEpoch.SetupID) + if err != nil { + return nil, fmt.Errorf("could not get next epoch setup: %w", err) + } + + for _, identity := range nextSetup.Participants { + exists := identities.Exists(identity) + + // add identity from next epoch that is not in current epoch + if !exists { + otherEpochIdentities = append(otherEpochIdentities, identity) + } + } + + default: + return nil, fmt.Errorf("invalid epoch phase: %s", phase) + } + + // add the identities from next/last epoch, with weight set to 0 + identities = append( + identities, + otherEpochIdentities.Map(mapfunc.WithWeight(0))..., + ) + + // apply the filter to the participants + identities = identities.Filter(selector) + + // apply a deterministic sort to the participants + identities = identities.Sort(flow.Canonical) + + return identities, nil +} + +func (s *Snapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error) { + // filter identities at snapshot for node ID + identities, err := s.Identities(filter.HasNodeID(nodeID)) + if err != nil { + return nil, fmt.Errorf("could not get identities: %w", err) + } + + // check if node ID is part of identities + if len(identities) == 0 { + return nil, protocol.IdentityNotFoundError{NodeID: nodeID} + } + return identities[0], nil +} + +// Commit retrieves the latest execution state commitment at the current block snapshot. This +// commitment represents the execution state as currently finalized. +func (s *Snapshot) Commit() (flow.StateCommitment, error) { + // get the ID of the sealed block + seal, err := s.state.seals.HighestInFork(s.blockID) + if err != nil { + return flow.DummyStateCommitment, fmt.Errorf("could not retrieve sealed state commit: %w", err) + } + return seal.FinalState, nil +} + +func (s *Snapshot) SealedResult() (*flow.ExecutionResult, *flow.Seal, error) { + seal, err := s.state.seals.HighestInFork(s.blockID) + if err != nil { + return nil, nil, fmt.Errorf("could not look up latest seal: %w", err) + } + result, err := s.state.results.ByID(seal.ResultID) + if err != nil { + return nil, nil, fmt.Errorf("could not get latest result: %w", err) + } + return result, seal, nil +} + +// SealingSegment will walk through the chain backward until we reach the block referenced +// by the latest seal and build a SealingSegment. As we visit each block we check each execution +// receipt in the block's payload to make sure we have a corresponding execution result, any +// execution results missing from blocks are stored in the `SealingSegment.ExecutionResults` field. +// See `model/flow/sealing_segment.md` for detailed technical specification of the Sealing Segment +// +// Expected errors during normal operations: +// - protocol.ErrSealingSegmentBelowRootBlock if sealing segment would stretch beyond the node's local history cut-off +// - protocol.UnfinalizedSealingSegmentError if sealing segment would contain unfinalized blocks (including orphaned blocks) +func (s *Snapshot) SealingSegment() (*flow.SealingSegment, error) { + // Lets denote the highest block in the sealing segment `head` (initialized below). + // Based on the tech spec `flow/sealing_segment.md`, the Sealing Segment must contain contain + // enough history to satisfy _all_ of the following conditions: + // (i) The highest sealed block as of `head` needs to be included in the sealing segment. + // This is relevant if `head` does not contain any seals. + // (ii) All blocks that are sealed by `head`. This is relevant if head` contains _multiple_ seals. + // (iii) The sealing segment should contain the history back to (including): + // limitHeight := max(blockSealedAtHead.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + // Per convention, we include the blocks for (i) in the `SealingSegment.Blocks`, while the + // additional blocks for (ii) and optionally (iii) are contained in as `SealingSegment.ExtraBlocks`. + head, err := s.state.blocks.ByID(s.blockID) + if err != nil { + return nil, fmt.Errorf("could not get snapshot's reference block: %w", err) + } + if head.Header.Height < s.state.finalizedRootHeight { + return nil, protocol.ErrSealingSegmentBelowRootBlock + } + + // Verify that head of sealing segment is finalized. + finalizedBlockAtHeight, err := s.state.headers.BlockIDByHeight(head.Header.Height) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, protocol.NewUnfinalizedSealingSegmentErrorf("head of sealing segment at height %d is not finalized: %w", head.Header.Height, err) + } + return nil, fmt.Errorf("exception while retrieving finzalized bloc, by height: %w", err) + } + if finalizedBlockAtHeight != s.blockID { // comparison of fixed-length arrays + return nil, protocol.NewUnfinalizedSealingSegmentErrorf("head of sealing segment is orphaned, finalized block at height %d is %x", head.Header.Height, finalizedBlockAtHeight) + } + + // STEP (i): highest sealed block as of `head` must be included. + seal, err := s.state.seals.HighestInFork(s.blockID) + if err != nil { + return nil, fmt.Errorf("could not get seal for sealing segment: %w", err) + } + blockSealedAtHead, err := s.state.headers.ByBlockID(seal.BlockID) + if err != nil { + return nil, fmt.Errorf("could not get block: %w", err) + } + + // walk through the chain backward until we reach the block referenced by + // the latest seal - the returned segment includes this block + builder := flow.NewSealingSegmentBuilder(s.state.results.ByID, s.state.seals.HighestInFork) + scraper := func(header *flow.Header) error { + blockID := header.ID() + block, err := s.state.blocks.ByID(blockID) + if err != nil { + return fmt.Errorf("could not get block: %w", err) + } + + err = builder.AddBlock(block) + if err != nil { + return fmt.Errorf("could not add block to sealing segment: %w", err) + } + + return nil + } + err = fork.TraverseForward(s.state.headers, s.blockID, scraper, fork.IncludingBlock(seal.BlockID)) + if err != nil { + return nil, fmt.Errorf("could not traverse sealing segment: %w", err) + } + + // STEP (ii): extend history down to the lowest block, whose seal is included in `head` + lowestSealedByHead := blockSealedAtHead + for _, sealInHead := range head.Payload.Seals { + h, e := s.state.headers.ByBlockID(sealInHead.BlockID) + if e != nil { + return nil, fmt.Errorf("could not get block (id=%x) for seal: %w", seal.BlockID, e) // storage.ErrNotFound or exception + } + if h.Height < lowestSealedByHead.Height { + lowestSealedByHead = h + } + } + + // STEP (iii): extended history to allow checking for duplicated collections, i.e. + // limitHeight = max(blockSealedAtHead.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + limitHeight := s.state.sporkRootBlockHeight + if blockSealedAtHead.Height > s.state.sporkRootBlockHeight+flow.DefaultTransactionExpiry { + limitHeight = blockSealedAtHead.Height - flow.DefaultTransactionExpiry + } + + // As we have to satisfy (ii) _and_ (iii), we have to take the longest history, i.e. the lowest height. + if lowestSealedByHead.Height < limitHeight { + limitHeight = lowestSealedByHead.Height + if limitHeight < s.state.sporkRootBlockHeight { // sanity check; should never happen + return nil, fmt.Errorf("unexpected internal error: calculated history-cutoff at height %d, which is lower than the spork's root height %d", limitHeight, s.state.sporkRootBlockHeight) + } + } + if limitHeight < blockSealedAtHead.Height { + // we need to include extra blocks in sealing segment + extraBlocksScraper := func(header *flow.Header) error { + blockID := header.ID() + block, err := s.state.blocks.ByID(blockID) + if err != nil { + return fmt.Errorf("could not get block: %w", err) + } + + err = builder.AddExtraBlock(block) + if err != nil { + return fmt.Errorf("could not add block to sealing segment: %w", err) + } + + return nil + } + + err = fork.TraverseBackward(s.state.headers, blockSealedAtHead.ParentID, extraBlocksScraper, fork.IncludingHeight(limitHeight)) + if err != nil { + return nil, fmt.Errorf("could not traverse extra blocks for sealing segment: %w", err) + } + } + + segment, err := builder.SealingSegment() + if err != nil { + return nil, fmt.Errorf("could not build sealing segment: %w", err) + } + + return segment, nil +} + +func (s *Snapshot) Descendants() ([]flow.Identifier, error) { + descendants, err := s.descendants(s.blockID) + if err != nil { + return nil, fmt.Errorf("failed to traverse the descendants tree of block %v: %w", s.blockID, err) + } + return descendants, nil +} + +func (s *Snapshot) lookupChildren(blockID flow.Identifier) ([]flow.Identifier, error) { + var children flow.IdentifierList + err := s.state.db.View(procedure.LookupBlockChildren(blockID, &children)) + if err != nil { + return nil, fmt.Errorf("could not get children of block %v: %w", blockID, err) + } + return children, nil +} + +func (s *Snapshot) descendants(blockID flow.Identifier) ([]flow.Identifier, error) { + descendantIDs, err := s.lookupChildren(blockID) + if err != nil { + return nil, err + } + + for _, descendantID := range descendantIDs { + additionalIDs, err := s.descendants(descendantID) + if err != nil { + return nil, err + } + descendantIDs = append(descendantIDs, additionalIDs...) + } + return descendantIDs, nil +} + +// RandomSource returns the seed for the current block's snapshot. +// Expected error returns: +// * storage.ErrNotFound is returned if the QC is unknown. +func (s *Snapshot) RandomSource() ([]byte, error) { + qc, err := s.QuorumCertificate() + if err != nil { + return nil, err + } + randomSource, err := model.BeaconSignature(qc) + if err != nil { + return nil, fmt.Errorf("could not create seed from QC's signature: %w", err) + } + return randomSource, nil +} + +func (s *Snapshot) Epochs() protocol.EpochQuery { + return &EpochQuery{ + snap: s, + } +} + +func (s *Snapshot) Params() protocol.GlobalParams { + return s.state.Params() +} + +func (s *Snapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { + head, err := s.state.headers.ByBlockID(s.blockID) + if err != nil { + return nil, err + } + + return s.state.versionBeacons.Highest(head.Height) +} + +// EpochQuery encapsulates querying epochs w.r.t. a snapshot. +type EpochQuery struct { + snap *Snapshot +} + +// Current returns the current epoch. +func (q *EpochQuery) Current() protocol.Epoch { + // all errors returned from storage reads here are unexpected, because all + // snapshots reside within a current epoch, which must be queryable + status, err := q.snap.state.epoch.statuses.ByBlockID(q.snap.blockID) + if err != nil { + return invalid.NewEpochf("could not get epoch status for block %x: %w", q.snap.blockID, err) + } + setup, err := q.snap.state.epoch.setups.ByID(status.CurrentEpoch.SetupID) + if err != nil { + return invalid.NewEpochf("could not get current EpochSetup (id=%x) for block %x: %w", status.CurrentEpoch.SetupID, q.snap.blockID, err) + } + commit, err := q.snap.state.epoch.commits.ByID(status.CurrentEpoch.CommitID) + if err != nil { + return invalid.NewEpochf("could not get current EpochCommit (id=%x) for block %x: %w", status.CurrentEpoch.CommitID, q.snap.blockID, err) + } + + firstHeight, _, epochStarted, _, err := q.retrieveEpochHeightBounds(setup.Counter) + if err != nil { + return invalid.NewEpochf("could not get current epoch height bounds: %s", err.Error()) + } + if epochStarted { + return inmem.NewStartedEpoch(setup, commit, firstHeight) + } + return inmem.NewCommittedEpoch(setup, commit) +} + +// Next returns the next epoch, if it is available. +func (q *EpochQuery) Next() protocol.Epoch { + + status, err := q.snap.state.epoch.statuses.ByBlockID(q.snap.blockID) + if err != nil { + return invalid.NewEpochf("could not get epoch status for block %x: %w", q.snap.blockID, err) + } + phase, err := status.Phase() + if err != nil { + // critical error: malformed EpochStatus in storage + return invalid.NewEpochf("read malformed EpochStatus from storage: %w", err) + } + // if we are in the staking phase, the next epoch is not setup yet + if phase == flow.EpochPhaseStaking { + return invalid.NewEpoch(protocol.ErrNextEpochNotSetup) + } + + // if we are in setup phase, return a SetupEpoch + nextSetup, err := q.snap.state.epoch.setups.ByID(status.NextEpoch.SetupID) + if err != nil { + // all errors are critical, because we must be able to retrieve EpochSetup when in setup phase + return invalid.NewEpochf("could not get next EpochSetup (id=%x) for block %x: %w", status.NextEpoch.SetupID, q.snap.blockID, err) + } + if phase == flow.EpochPhaseSetup { + return inmem.NewSetupEpoch(nextSetup) + } + + // if we are in committed phase, return a CommittedEpoch + nextCommit, err := q.snap.state.epoch.commits.ByID(status.NextEpoch.CommitID) + if err != nil { + // all errors are critical, because we must be able to retrieve EpochCommit when in committed phase + return invalid.NewEpochf("could not get next EpochCommit (id=%x) for block %x: %w", status.NextEpoch.CommitID, q.snap.blockID, err) + } + return inmem.NewCommittedEpoch(nextSetup, nextCommit) +} + +// Previous returns the previous epoch. During the first epoch after the root +// block, this returns a sentinel error (since there is no previous epoch). +// For all other epochs, returns the previous epoch. +func (q *EpochQuery) Previous() protocol.Epoch { + + status, err := q.snap.state.epoch.statuses.ByBlockID(q.snap.blockID) + if err != nil { + return invalid.NewEpochf("could not get epoch status for block %x: %w", q.snap.blockID, err) + } + + // CASE 1: there is no previous epoch - this indicates we are in the first + // epoch after a spork root or genesis block + if !status.HasPrevious() { + return invalid.NewEpoch(protocol.ErrNoPreviousEpoch) + } + + // CASE 2: we are in any other epoch - retrieve the setup and commit events + // for the previous epoch + setup, err := q.snap.state.epoch.setups.ByID(status.PreviousEpoch.SetupID) + if err != nil { + // all errors are critical, because we must be able to retrieve EpochSetup for previous epoch + return invalid.NewEpochf("could not get previous EpochSetup (id=%x) for block %x: %w", status.PreviousEpoch.SetupID, q.snap.blockID, err) + } + commit, err := q.snap.state.epoch.commits.ByID(status.PreviousEpoch.CommitID) + if err != nil { + // all errors are critical, because we must be able to retrieve EpochCommit for previous epoch + return invalid.NewEpochf("could not get current EpochCommit (id=%x) for block %x: %w", status.PreviousEpoch.CommitID, q.snap.blockID, err) + } + + firstHeight, finalHeight, _, epochEnded, err := q.retrieveEpochHeightBounds(setup.Counter) + if err != nil { + return invalid.NewEpochf("could not get epoch height bounds: %w", err) + } + if epochEnded { + return inmem.NewEndedEpoch(setup, commit, firstHeight, finalHeight) + } + return inmem.NewStartedEpoch(setup, commit, firstHeight) +} + +// retrieveEpochHeightBounds retrieves the height bounds for an epoch. +// Height bounds are NOT fork-aware, and are only determined upon finalization. +// +// Since the protocol state's API is fork-aware, we may be querying an +// un-finalized block, as in the following example: +// +// Epoch 1 Epoch 2 +// A <- B <-|- C <- D +// +// Suppose block B is the latest finalized block and we have queried block D. +// Then, the transition from epoch 1 to 2 has not been committed, because the first block of epoch 2 has not been finalized. +// In this case, the final block of Epoch 1, from the perspective of block D, is unknown. +// There are edge-case scenarios, where a different fork could exist (as illustrated below) +// that still adds additional blocks to Epoch 1. +// +// Epoch 1 Epoch 2 +// A <- B <---|-- C <- D +// ^ +// ╰ X <-|- X <- Y <- Z +// +// Returns: +// - (0, 0, false, false, nil) if epoch is not started +// - (firstHeight, 0, true, false, nil) if epoch is started but not ended +// - (firstHeight, finalHeight, true, true, nil) if epoch is ended +// +// No errors are expected during normal operation. +func (q *EpochQuery) retrieveEpochHeightBounds(epoch uint64) (firstHeight, finalHeight uint64, isFirstBlockFinalized, isLastBlockFinalized bool, err error) { + err = q.snap.state.db.View(func(tx *badger.Txn) error { + // Retrieve the epoch's first height + err = operation.RetrieveEpochFirstHeight(epoch, &firstHeight)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + isFirstBlockFinalized = false + isLastBlockFinalized = false + return nil + } + return err // unexpected error + } + isFirstBlockFinalized = true + + var subsequentEpochFirstHeight uint64 + err = operation.RetrieveEpochFirstHeight(epoch+1, &subsequentEpochFirstHeight)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + isLastBlockFinalized = false + return nil + } + return err // unexpected error + } + finalHeight = subsequentEpochFirstHeight - 1 + isLastBlockFinalized = true + + return nil + }) + if err != nil { + return 0, 0, false, false, err + } + return firstHeight, finalHeight, isFirstBlockFinalized, isLastBlockFinalized, nil +} diff --git a/state/protocol/pebble/snapshot_test.go b/state/protocol/pebble/snapshot_test.go new file mode 100644 index 00000000000..9b6f783ce0e --- /dev/null +++ b/state/protocol/pebble/snapshot_test.go @@ -0,0 +1,1437 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger_test + +import ( + "context" + "errors" + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/factory" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module/signature" + statepkg "github.com/onflow/flow-go/state" + "github.com/onflow/flow-go/state/protocol" + bprotocol "github.com/onflow/flow-go/state/protocol/badger" + "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/prg" + "github.com/onflow/flow-go/state/protocol/util" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestUnknownReferenceBlock tests queries for snapshots which should be unknown. +// We use this fixture: +// - Root height: 100 +// - Heights [100, 110] are finalized +// - Height 111 is unfinalized +func TestUnknownReferenceBlock(t *testing.T) { + rootHeight := uint64(100) + participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(participants, func(block *flow.Block) { + block.Header.Height = rootHeight + }) + + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + // build some finalized non-root blocks (heights 101-110) + head := rootSnapshot.Encodable().Head + const nBlocks = 10 + for i := 0; i < nBlocks; i++ { + next := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, next) + head = next.Header + } + // build an unfinalized block (height 111) + buildBlock(t, state, unittest.BlockWithParentFixture(head)) + + finalizedHeader, err := state.Final().Head() + require.NoError(t, err) + + t.Run("below root height", func(t *testing.T) { + _, err := state.AtHeight(rootHeight - 1).Head() + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + }) + t.Run("above finalized height, non-existent height", func(t *testing.T) { + _, err := state.AtHeight(finalizedHeader.Height + 100).Head() + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + }) + t.Run("above finalized height, existent height", func(t *testing.T) { + _, err := state.AtHeight(finalizedHeader.Height + 1).Head() + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + }) + t.Run("unknown block ID", func(t *testing.T) { + _, err := state.AtBlockID(unittest.IdentifierFixture()).Head() + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + }) + }) +} + +func TestHead(t *testing.T) { + participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.State) { + + t.Run("works with block number", func(t *testing.T) { + retrieved, err := state.AtHeight(head.Height).Head() + require.NoError(t, err) + require.Equal(t, head.ID(), retrieved.ID()) + }) + + t.Run("works with block id", func(t *testing.T) { + retrieved, err := state.AtBlockID(head.ID()).Head() + require.NoError(t, err) + require.Equal(t, head.ID(), retrieved.ID()) + }) + + t.Run("works with finalized block", func(t *testing.T) { + retrieved, err := state.Final().Head() + require.NoError(t, err) + require.Equal(t, head.ID(), retrieved.ID()) + }) + }) +} + +// TestSnapshot_Params tests retrieving global protocol state parameters from +// a protocol state snapshot. +func TestSnapshot_Params(t *testing.T) { + participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(participants) + + expectedChainID, err := rootSnapshot.Params().ChainID() + require.NoError(t, err) + expectedSporkID, err := rootSnapshot.Params().SporkID() + require.NoError(t, err) + expectedProtocolVersion, err := rootSnapshot.Params().ProtocolVersion() + require.NoError(t, err) + + rootHeader, err := rootSnapshot.Head() + require.NoError(t, err) + + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + // build some non-root blocks + head := rootHeader + const nBlocks = 10 + for i := 0; i < nBlocks; i++ { + next := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, next) + head = next.Header + } + + // test params from both root, final, and in between + snapshots := []protocol.Snapshot{ + state.AtHeight(0), + state.AtHeight(uint64(rand.Intn(nBlocks))), + state.Final(), + } + for _, snapshot := range snapshots { + t.Run("should be able to get chain ID from snapshot", func(t *testing.T) { + chainID, err := snapshot.Params().ChainID() + require.NoError(t, err) + assert.Equal(t, expectedChainID, chainID) + }) + t.Run("should be able to get spork ID from snapshot", func(t *testing.T) { + sporkID, err := snapshot.Params().SporkID() + require.NoError(t, err) + assert.Equal(t, expectedSporkID, sporkID) + }) + t.Run("should be able to get protocol version from snapshot", func(t *testing.T) { + protocolVersion, err := snapshot.Params().ProtocolVersion() + require.NoError(t, err) + assert.Equal(t, expectedProtocolVersion, protocolVersion) + }) + } + }) +} + +// TestSnapshot_Descendants builds a sample chain with next structure: +// +// A (finalized) <- B <- C <- D <- E <- F +// <- G <- H <- I <- J +// +// snapshot.Descendants has to return [B, C, D, E, F, G, H, I, J]. +func TestSnapshot_Descendants(t *testing.T) { + participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(participants) + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + var expectedBlocks []flow.Identifier + for i := 5; i > 3; i-- { + for _, block := range unittest.ChainFixtureFrom(i, head) { + err := state.Extend(context.Background(), block) + require.NoError(t, err) + expectedBlocks = append(expectedBlocks, block.ID()) + } + } + + pendingBlocks, err := state.AtBlockID(head.ID()).Descendants() + require.NoError(t, err) + require.ElementsMatch(t, expectedBlocks, pendingBlocks) + }) +} + +func TestIdentities(t *testing.T) { + identities := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(identities) + util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.State) { + + t.Run("no filter", func(t *testing.T) { + actual, err := state.Final().Identities(filter.Any) + require.NoError(t, err) + assert.ElementsMatch(t, identities, actual) + }) + + t.Run("single identity", func(t *testing.T) { + expected := identities[rand.Intn(len(identities))] + actual, err := state.Final().Identity(expected.NodeID) + require.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("filtered", func(t *testing.T) { + sample, err := identities.SamplePct(0.1) + require.NoError(t, err) + filters := []flow.IdentityFilter{ + filter.HasRole(flow.RoleCollection), + filter.HasNodeID(sample.NodeIDs()...), + filter.HasWeight(true), + } + + for _, filterfunc := range filters { + expected := identities.Filter(filterfunc) + actual, err := state.Final().Identities(filterfunc) + require.NoError(t, err) + assert.ElementsMatch(t, expected, actual) + } + }) + }) +} + +func TestClusters(t *testing.T) { + nClusters := 3 + nCollectors := 7 + + collectors := unittest.IdentityListFixture(nCollectors, unittest.WithRole(flow.RoleCollection)) + identities := append(unittest.IdentityListFixture(4, unittest.WithAllRolesExcept(flow.RoleCollection)), collectors...) + + root, result, seal := unittest.BootstrapFixture(identities) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + setup.Assignments = unittest.ClusterAssignment(uint(nClusters), collectors) + clusterQCs := unittest.QuorumCertificatesFromAssignments(setup.Assignments) + commit.ClusterQCs = flow.ClusterQCVoteDatasFromQCs(clusterQCs) + seal.ResultID = result.ID() + + rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) + require.NoError(t, err) + + util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.State) { + expectedClusters, err := factory.NewClusterList(setup.Assignments, collectors) + require.NoError(t, err) + actualClusters, err := state.Final().Epochs().Current().Clustering() + require.NoError(t, err) + + require.Equal(t, nClusters, len(expectedClusters)) + require.Equal(t, len(expectedClusters), len(actualClusters)) + + for i := 0; i < nClusters; i++ { + expected := expectedClusters[i] + actual := actualClusters[i] + + assert.Equal(t, len(expected), len(actual)) + assert.Equal(t, expected.ID(), actual.ID()) + } + }) +} + +// TestSealingSegment tests querying sealing segment with respect to various snapshots. +// +// For each valid sealing segment, we also test bootstrapping with this sealing segment. +func TestSealingSegment(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + head, err := rootSnapshot.Head() + require.NoError(t, err) + + t.Run("root sealing segment", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + expected, err := rootSnapshot.SealingSegment() + require.NoError(t, err) + actual, err := state.AtBlockID(head.ID()).SealingSegment() + require.NoError(t, err) + + assert.Len(t, actual.ExecutionResults, 1) + assert.Len(t, actual.Blocks, 1) + assert.Empty(t, actual.ExtraBlocks) + unittest.AssertEqualBlocksLenAndOrder(t, expected.Blocks, actual.Blocks) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(head.ID())) + }) + }) + + // test sealing segment for non-root segment where the latest seal is the + // root seal, but the segment contains more than the root block. + // ROOT <- B1 + // Expected sealing segment: [ROOT, B1], extra blocks: [] + t.Run("non-root with root seal as latest seal", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // build an extra block on top of root + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + segment, err := state.AtBlockID(block1.ID()).SealingSegment() + require.NoError(t, err) + + // build a valid child B2 to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block1.Header)) + + // sealing segment should contain B1 and B2 + // B2 is reference of snapshot, B1 is latest sealed + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{rootSnapshot.Encodable().SealingSegment.Sealed(), block1}, segment.Blocks) + assert.Len(t, segment.ExecutionResults, 1) + assert.Empty(t, segment.ExtraBlocks) + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block1.ID())) + }) + }) + + // test sealing segment for non-root segment with simple sealing structure + // (no blocks in between reference block and latest sealed) + // ROOT <- B1 <- B2(R1) <- B3(S1) + // Expected sealing segment: [B1, B2, B3], extra blocks: [ROOT] + t.Run("non-root", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // build a block to seal + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + // build a block sealing block1 + block3 := unittest.BlockWithParentFixture(block2.Header) + + block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block3) + + segment, err := state.AtBlockID(block3.ID()).SealingSegment() + require.NoError(t, err) + + require.Len(t, segment.ExtraBlocks, 1) + assert.Equal(t, segment.ExtraBlocks[0].Header.Height, head.Height) + + // build a valid child B3 to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block3.Header)) + + // sealing segment should contain B1, B2, B3 + // B3 is reference of snapshot, B1 is latest sealed + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1, block2, block3}, segment.Blocks) + assert.Len(t, segment.ExecutionResults, 1) + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block3.ID())) + }) + }) + + // test sealing segment for sealing segment with a large number of blocks + // between the reference block and latest sealed + // ROOT <- B1 <- .... <- BN(S1) + // Expected sealing segment: [B1, ..., BN], extra blocks: [ROOT] + t.Run("long sealing segment", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + + // build a block to seal + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + parent := block1 + // build a large chain of intermediary blocks + for i := 0; i < 100; i++ { + next := unittest.BlockWithParentFixture(parent.Header) + if i == 0 { + // Repetitions of the same receipt in one fork would be a protocol violation. + // Hence, we include the result only once in the direct child of B1. + next.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + } + buildFinalizedBlock(t, state, next) + parent = next + } + + // build the block sealing block 1 + blockN := unittest.BlockWithParentFixture(parent.Header) + + blockN.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, blockN) + + segment, err := state.AtBlockID(blockN.ID()).SealingSegment() + require.NoError(t, err) + + assert.Len(t, segment.ExecutionResults, 1) + // sealing segment should cover range [B1, BN] + assert.Len(t, segment.Blocks, 102) + assert.Len(t, segment.ExtraBlocks, 1) + assert.Equal(t, segment.ExtraBlocks[0].Header.Height, head.Height) + // first and last blocks should be B1, BN + assert.Equal(t, block1.ID(), segment.Blocks[0].ID()) + assert.Equal(t, blockN.ID(), segment.Blocks[101].ID()) + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(blockN.ID())) + }) + }) + + // test sealing segment where the segment blocks contain seals for + // ancestor blocks prior to the sealing segment + // ROOT <- B1 <- B2(R1) <- B3 <- B4(R2, S1) <- B5 <- B6(S2) + // Expected sealing segment: [B2, B3, B4], Extra blocks: [ROOT, B1] + t.Run("overlapping sealing segment", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + buildFinalizedBlock(t, state, block3) + + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block4) + + block5 := unittest.BlockWithParentFixture(block4.Header) + buildFinalizedBlock(t, state, block5) + + block6 := unittest.BlockWithParentFixture(block5.Header) + block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) + buildFinalizedBlock(t, state, block6) + + segment, err := state.AtBlockID(block6.ID()).SealingSegment() + require.NoError(t, err) + + // build a valid child to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block6.Header)) + + // sealing segment should be [B2, B3, B4, B5, B6] + require.Len(t, segment.Blocks, 5) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5, block6}, segment.Blocks) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1}, segment.ExtraBlocks[1:]) + require.Len(t, segment.ExecutionResults, 1) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block6.ID())) + }) + }) + + // test sealing segment where you have a chain that is 5 blocks long and the block 5 has a seal for block 2 + // block 2 also contains a receipt but no result. + // ROOT -> B1(Result_A, Receipt_A_1) -> B2(Result_B, Receipt_B, Receipt_A_2) -> B3(Receipt_C, Result_C) -> B4 -> B5(Seal_C) + // the segment for B5 should be `[B2,B3,B4,B5] + [Result_A]` + t.Run("sealing segment with 4 blocks and 1 execution result decoupled", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // simulate scenario where execution result is missing from block payload + // SealingSegment() should get result from results db and store it on ExecutionReceipts + // field on SealingSegment + resultA := unittest.ExecutionResultFixture() + receiptA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultA)) + receiptA2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultA)) + + // receipt b also contains result b + receiptB := unittest.ExecutionReceiptFixture() + + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptA1))) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptB), unittest.WithReceiptsAndNoResults(receiptA2))) + receiptC, sealC := unittest.ReceiptAndSealForBlock(block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptC))) + + block4 := unittest.BlockWithParentFixture(block3.Header) + + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(sealC))) + + buildFinalizedBlock(t, state, block1) + buildFinalizedBlock(t, state, block2) + buildFinalizedBlock(t, state, block3) + buildFinalizedBlock(t, state, block4) + buildFinalizedBlock(t, state, block5) + + segment, err := state.AtBlockID(block5.ID()).SealingSegment() + require.NoError(t, err) + + // build a valid child to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + + require.Len(t, segment.Blocks, 4) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5}, segment.Blocks) + require.Contains(t, segment.ExecutionResults, resultA) + require.Len(t, segment.ExecutionResults, 2) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block5.ID())) + }) + }) + + // test sealing segment where you have a chain that is 5 blocks long and the block 5 has a seal for block 2. + // even though block2 & block3 both reference ResultA it should be added to the segment execution results list once. + // block3 also references ResultB, so it should exist in the segment execution results as well. + // root -> B1[Result_A, Receipt_A_1] -> B2[Result_B, Receipt_B, Receipt_A_2] -> B3[Receipt_B_2, Receipt_for_seal, Receipt_A_3] -> B4 -> B5 (Seal_B2) + t.Run("sealing segment with 4 blocks and 2 execution result decoupled", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // simulate scenario where execution result is missing from block payload + // SealingSegment() should get result from results db and store it on ExecutionReceipts + // field on SealingSegment + resultA := unittest.ExecutionResultFixture() + + // 3 execution receipts for Result_A + receiptA1 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultA)) + receiptA2 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultA)) + receiptA3 := unittest.ExecutionReceiptFixture(unittest.WithResult(resultA)) + + // receipt b also contains result b + receiptB := unittest.ExecutionReceiptFixture() + // get second receipt for Result_B, now we have 2 receipts for a single execution result + receiptB2 := unittest.ExecutionReceiptFixture(unittest.WithResult(&receiptB.ExecutionResult)) + + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptA1))) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptB), unittest.WithReceiptsAndNoResults(receiptA2))) + + receiptForSeal, seal := unittest.ReceiptAndSealForBlock(block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptForSeal), unittest.WithReceiptsAndNoResults(receiptB2, receiptA3))) + + block4 := unittest.BlockWithParentFixture(block3.Header) + + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) + + buildFinalizedBlock(t, state, block1) + buildFinalizedBlock(t, state, block2) + buildFinalizedBlock(t, state, block3) + buildFinalizedBlock(t, state, block4) + buildFinalizedBlock(t, state, block5) + + segment, err := state.AtBlockID(block5.ID()).SealingSegment() + require.NoError(t, err) + + // build a valid child to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + + require.Len(t, segment.Blocks, 4) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5}, segment.Blocks) + require.Contains(t, segment.ExecutionResults, resultA) + // ResultA should only be added once even though it is referenced in 2 different blocks + require.Len(t, segment.ExecutionResults, 2) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block5.ID())) + }) + }) + + // Test the case where the reference block of the snapshot contains no seal. + // We should consider the latest seal in a prior block. + // ROOT <- B1 <- B2(R1) <- B3 <- B4(S1) <- B5 + // Expected sealing segment: [B1, B2, B3, B4, B5], Extra blocks: [ROOT] + t.Run("sealing segment where highest block in segment does not seal lowest", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // build a block to seal + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + // build a block sealing block1 + block2 := unittest.BlockWithParentFixture(block1.Header) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + buildFinalizedBlock(t, state, block3) + + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block4) + + block5 := unittest.BlockWithParentFixture(block4.Header) + buildFinalizedBlock(t, state, block5) + + snapshot := state.AtBlockID(block5.ID()) + + // build a valid child to ensure we have a QC + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + // sealing segment should contain B1 and B5 + // B5 is reference of snapshot, B1 is latest sealed + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1, block2, block3, block4, block5}, segment.Blocks) + assert.Len(t, segment.ExecutionResults, 1) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + }) + }) + + // Root <- B1 <- B2 <- ... <- B700(Seal_B699) + // Expected sealing segment: [B699, B700], Extra blocks: [B98, B99, ..., B698] + // where DefaultTransactionExpiry = 600 + t.Run("test extra blocks contain exactly DefaultTransactionExpiry number of blocks below the sealed block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + root := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, root) + + blocks := make([]*flow.Block, 0, flow.DefaultTransactionExpiry+3) + parent := root + for i := 0; i < flow.DefaultTransactionExpiry+1; i++ { + next := unittest.BlockWithParentFixture(parent.Header) + next.Header.View = next.Header.Height + 1 // set view so we are still in the same epoch + buildFinalizedBlock(t, state, next) + blocks = append(blocks, next) + parent = next + } + + // last sealed block + lastSealedBlock := parent + lastReceipt, lastSeal := unittest.ReceiptAndSealForBlock(lastSealedBlock) + prevLastBlock := unittest.BlockWithParentFixture(lastSealedBlock.Header) + prevLastBlock.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(lastReceipt), + )) + buildFinalizedBlock(t, state, prevLastBlock) + + // last finalized block + lastBlock := unittest.BlockWithParentFixture(prevLastBlock.Header) + lastBlock.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(lastSeal), + )) + buildFinalizedBlock(t, state, lastBlock) + + // build a valid child to ensure we have a QC + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(lastBlock.Header)) + + snapshot := state.AtBlockID(lastBlock.ID()) + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + + assert.Equal(t, lastBlock.Header, segment.Highest().Header) + assert.Equal(t, lastBlock.Header, segment.Finalized().Header) + assert.Equal(t, lastSealedBlock.Header, segment.Sealed().Header) + + // there are DefaultTransactionExpiry number of blocks in total + unittest.AssertEqualBlocksLenAndOrder(t, blocks[:flow.DefaultTransactionExpiry], segment.ExtraBlocks) + assert.Len(t, segment.ExtraBlocks, flow.DefaultTransactionExpiry) + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + + }) + }) + // Test the case where the reference block of the snapshot contains seals for blocks that are lower than the lowest sealing segment's block. + // This test case specifically checks if sealing segment includes both highest and lowest block sealed by head. + // ROOT <- B1 <- B2 <- B3(Seal_B1) <- B4 <- ... <- LastBlock(Seal_B2, Seal_B3, Seal_B4) + // Expected sealing segment: [B4, ..., B5], Extra blocks: [Root, B1, B2, B3] + t.Run("highest block seals outside segment", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // build a block to seal + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + // build a block sealing block1 + block2 := unittest.BlockWithParentFixture(block1.Header) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1), unittest.WithReceipts(receipt2))) + buildFinalizedBlock(t, state, block3) + + receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3))) + buildFinalizedBlock(t, state, block4) + + // build chain, so it's long enough to not target blocks as inside of flow.DefaultTransactionExpiry window. + parent := block4 + for i := 0; i < 1.5*flow.DefaultTransactionExpiry; i++ { + next := unittest.BlockWithParentFixture(parent.Header) + next.Header.View = next.Header.Height + 1 // set view so we are still in the same epoch + buildFinalizedBlock(t, state, next) + parent = next + } + + receipt4, seal4 := unittest.ReceiptAndSealForBlock(block4) + lastBlock := unittest.BlockWithParentFixture(parent.Header) + lastBlock.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2, seal3, seal4), unittest.WithReceipts(receipt4))) + buildFinalizedBlock(t, state, lastBlock) + + snapshot := state.AtBlockID(lastBlock.ID()) + + // build a valid child to ensure we have a QC + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(lastBlock.Header)) + + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + assert.Equal(t, lastBlock.Header, segment.Highest().Header) + assert.Equal(t, block4.Header, segment.Sealed().Header) + root := rootSnapshot.Encodable().SealingSegment.Sealed() + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{root, block1, block2, block3}, segment.ExtraBlocks) + assert.Len(t, segment.ExecutionResults, 2) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + }) + }) +} + +// TestSealingSegment_FailureCases verifies that SealingSegment construction fails with expected sentinel +// errors in case the caller violates the API contract: +// 1. The lowest block that can serve as head of a SealingSegment is the node's local root block. +// 2. Unfinalized blocks cannot serve as head of a SealingSegment. There are two distinct sub-cases: +// (2a) A pending block is chosen as head; at this height no block has been finalized. +// (2b) An orphaned block is chosen as head; at this height a block other than the orphaned has been finalized. +func TestSealingSegment_FailureCases(t *testing.T) { + sporkRootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + sporkRoot, err := sporkRootSnapshot.Head() + require.NoError(t, err) + + // SCENARIO 1. + // Here, we want to specifically test correct handling of the edge case, where a block exists in storage + // that has _lower height_ than the node's local root block. Such blocks are typically contained in the + // bootstrapping data, such that all entities referenced in the local root block can be resolved. + // Is is possible to retrieve blocks that are lower than the local root block from storage, directly + // via their ID. Despite these blocks existing in storage, SealingSegment construction should be + // because the known history is potentially insufficient when going below the root block. + t.Run("sealing segment from block below local state root", func(t *testing.T) { + // Step I: constructing bootstrapping snapshot with some short history: + // + // ╭───── finalized blocks ─────╮ + // <- b1 <- b2(result(b1)) <- b3(seal(b1)) <- + // └── head ──┘ + // + b1 := unittest.BlockWithParentFixture(sporkRoot) // construct block b1, append to state and finalize + receipt, seal := unittest.ReceiptAndSealForBlock(b1) + b2 := unittest.BlockWithParentFixture(b1.Header) // construct block b2, append to state and finalize + b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt))) + b3 := unittest.BlockWithParentFixture(b2.Header) // construct block b3 with seal for b1, append it to state and finalize + b3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) + + multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + for _, b := range []*flow.Block{b1, b2, b3} { + buildFinalizedBlock(t, state, b) + } + b4 := unittest.BlockWithParentFixture(b3.Header) + require.NoError(t, state.ExtendCertified(context.Background(), b4, unittest.CertifyBlock(b4.Header))) // add child of b3 to ensure we have a QC for b3 + return state.AtBlockID(b3.ID()) + }) + + // Step 2: bootstrapping new state based on sealing segment whose head is block b3. + // Thereby, the state should have b3 as its local root block. In addition, the blocks contained in the sealing + // segment, such as b2 should be stored in the state. + util.RunWithFollowerProtocolState(t, multipleBlockSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + localStateRootBlock, err := state.Params().FinalizedRoot() + require.NoError(t, err) + assert.Equal(t, b3.ID(), localStateRootBlock.ID()) + + // verify that b2 is known to the protocol state, but constructing a sealing segment fails + _, err = state.AtBlockID(b2.ID()).Head() + require.NoError(t, err) + _, err = state.AtBlockID(b2.ID()).SealingSegment() + assert.ErrorIs(t, err, protocol.ErrSealingSegmentBelowRootBlock) + + // lowest block that allows for sealing segment construction is root block: + _, err = state.AtBlockID(b3.ID()).SealingSegment() + require.NoError(t, err) + }) + }) + + // SCENARIO 2a: A pending block is chosen as head; at this height no block has been finalized. + t.Run("sealing segment from unfinalized, pending block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // add _unfinalized_ blocks b1 and b2 to state (block b5 is necessary, so b1 has a QC, which is a consistency requirement for subsequent finality) + b1 := unittest.BlockWithParentFixture(sporkRoot) + b2 := unittest.BlockWithParentFixture(b1.Header) + require.NoError(t, state.ExtendCertified(context.Background(), b1, b2.Header.QuorumCertificate())) + require.NoError(t, state.ExtendCertified(context.Background(), b2, unittest.CertifyBlock(b2.Header))) // adding block b5 (providing required QC for b1) + + // consistency check: there should be no finalized block in the protocol state at height `b1.Height` + _, err := state.AtHeight(b1.Header.Height).Head() // expect statepkg.ErrUnknownSnapshotReference as only finalized blocks are indexed by height + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + + // requesting a sealing segment from block b1 should fail, as b1 is not yet finalized + _, err = state.AtBlockID(b1.ID()).SealingSegment() + assert.True(t, protocol.IsUnfinalizedSealingSegmentError(err)) + }) + }) + + // SCENARIO 2b: An orphaned block is chosen as head; at this height a block other than the orphaned has been finalized. + t.Run("sealing segment from orphaned block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + orphaned := unittest.BlockWithParentFixture(sporkRoot) + orphanedChild := unittest.BlockWithParentFixture(orphaned.Header) + require.NoError(t, state.ExtendCertified(context.Background(), orphaned, orphanedChild.Header.QuorumCertificate())) + require.NoError(t, state.ExtendCertified(context.Background(), orphanedChild, unittest.CertifyBlock(orphanedChild.Header))) + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(sporkRoot)) + + // consistency check: the finalized block at height `orphaned.Height` should be different than `orphaned` + h, err := state.AtHeight(orphaned.Header.Height).Head() + require.NoError(t, err) + require.NotEqual(t, h.ID(), orphaned.ID()) + + // requesting a sealing segment from orphaned block should fail, as it is not finalized + _, err = state.AtBlockID(orphaned.ID()).SealingSegment() + assert.True(t, protocol.IsUnfinalizedSealingSegmentError(err)) + }) + }) + +} + +// TestBootstrapSealingSegmentWithExtraBlocks test sealing segment where the segment blocks contain collection +// guarantees referencing blocks prior to the sealing segment. After bootstrapping from sealing segment we should be able to +// extend with B7 with contains a guarantee referring B1. +// ROOT <- B1 <- B2(R1) <- B3 <- B4(S1) <- B5 <- B6(S2) +// Expected sealing segment: [B2, B3, B4, B5, B6], Extra blocks: [ROOT, B1] +func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + rootEpoch := rootSnapshot.Epochs().Current() + cluster, err := rootEpoch.Cluster(0) + require.NoError(t, err) + collID := cluster.Members()[0].NodeID + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + buildFinalizedBlock(t, state, block3) + + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block4) + + block5 := unittest.BlockWithParentFixture(block4.Header) + buildFinalizedBlock(t, state, block5) + + block6 := unittest.BlockWithParentFixture(block5.Header) + block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) + buildFinalizedBlock(t, state, block6) + + snapshot := state.AtBlockID(block6.ID()) + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + + // build a valid child to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block6.Header)) + + // sealing segment should be [B2, B3, B4, B5, B6] + require.Len(t, segment.Blocks, 5) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5, block6}, segment.Blocks) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1}, segment.ExtraBlocks[1:]) + require.Len(t, segment.ExecutionResults, 1) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + + // bootstrap from snapshot + util.RunWithFullProtocolState(t, snapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + block7 := unittest.BlockWithParentFixture(block6.Header) + guarantee := unittest.CollectionGuaranteeFixture(unittest.WithCollRef(block1.ID())) + guarantee.ChainID = cluster.ChainID() + + signerIndices, err := signature.EncodeSignersToIndices( + []flow.Identifier{collID}, []flow.Identifier{collID}) + require.NoError(t, err) + guarantee.SignerIndices = signerIndices + + block7.SetPayload(unittest.PayloadFixture(unittest.WithGuarantees(guarantee))) + buildBlock(t, state, block7) + }) + }) +} + +func TestLatestSealedResult(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + + t.Run("root snapshot", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + gotResult, gotSeal, err := state.Final().SealedResult() + require.NoError(t, err) + expectedResult, expectedSeal, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + assert.Equal(t, expectedResult.ID(), gotResult.ID()) + assert.Equal(t, expectedSeal, gotSeal) + }) + }) + + t.Run("non-root snapshot", func(t *testing.T) { + head, err := rootSnapshot.Head() + require.NoError(t, err) + + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + block1 := unittest.BlockWithParentFixture(head) + + block2 := unittest.BlockWithParentFixture(block1.Header) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt2, receipt3), + )) + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal2, seal3), + )) + + err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate()) + require.NoError(t, err) + + err = state.ExtendCertified(context.Background(), block3, block4.Header.QuorumCertificate()) + require.NoError(t, err) + + // B1 <- B2(R1) <- B3(S1) + // querying B3 should return result R1, seal S1 + t.Run("reference block contains seal", func(t *testing.T) { + gotResult, gotSeal, err := state.AtBlockID(block3.ID()).SealedResult() + require.NoError(t, err) + assert.Equal(t, block2.Payload.Results[0], gotResult) + assert.Equal(t, block3.Payload.Seals[0], gotSeal) + }) + + err = state.ExtendCertified(context.Background(), block4, block5.Header.QuorumCertificate()) + require.NoError(t, err) + + // B1 <- B2(S1) <- B3(S1) <- B4(R2,R3) + // querying B3 should still return (R1,S1) even though they are in parent block + t.Run("reference block contains no seal", func(t *testing.T) { + gotResult, gotSeal, err := state.AtBlockID(block4.ID()).SealedResult() + require.NoError(t, err) + assert.Equal(t, &receipt1.ExecutionResult, gotResult) + assert.Equal(t, seal1, gotSeal) + }) + + // B1 <- B2(R1) <- B3(S1) <- B4(R2,R3) <- B5(S2,S3) + // There are two seals in B5 - should return latest by height (S3,R3) + t.Run("reference block contains multiple seals", func(t *testing.T) { + err = state.ExtendCertified(context.Background(), block5, unittest.CertifyBlock(block5.Header)) + require.NoError(t, err) + + gotResult, gotSeal, err := state.AtBlockID(block5.ID()).SealedResult() + require.NoError(t, err) + assert.Equal(t, &receipt3.ExecutionResult, gotResult) + assert.Equal(t, seal3, gotSeal) + }) + }) + }) +} + +// test retrieving quorum certificate and seed +func TestQuorumCertificate(t *testing.T) { + identities := unittest.IdentityListFixture(5, unittest.WithAllRoles()) + rootSnapshot := unittest.RootSnapshotFixture(identities) + head, err := rootSnapshot.Head() + require.NoError(t, err) + + // should not be able to get QC or random beacon seed from a block with no children + t.Run("no QC available", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + + // create a block to query + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err := state.Extend(context.Background(), block1) + require.NoError(t, err) + + _, err = state.AtBlockID(block1.ID()).QuorumCertificate() + assert.ErrorIs(t, err, storage.ErrNotFound) + + _, err = state.AtBlockID(block1.ID()).RandomSource() + assert.ErrorIs(t, err, storage.ErrNotFound) + }) + }) + + // should be able to get QC and random beacon seed from root block + t.Run("root block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // since we bootstrap with a root snapshot, this will be the root block + _, err := state.AtBlockID(head.ID()).QuorumCertificate() + assert.NoError(t, err) + randomSeed, err := state.AtBlockID(head.ID()).RandomSource() + assert.NoError(t, err) + assert.Equal(t, len(randomSeed), prg.RandomSourceLength) + }) + }) + + // should be able to get QC and random beacon seed from a certified block + t.Run("follower-block-processable", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + + // add a block so we aren't testing against root + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + certifyingQC := unittest.CertifyBlock(block1.Header) + err := state.ExtendCertified(context.Background(), block1, certifyingQC) + require.NoError(t, err) + + // should be able to get QC/seed + qc, err := state.AtBlockID(block1.ID()).QuorumCertificate() + assert.NoError(t, err) + + assert.Equal(t, certifyingQC.SignerIndices, qc.SignerIndices) + assert.Equal(t, certifyingQC.SigData, qc.SigData) + assert.Equal(t, block1.Header.View, qc.View) + + _, err = state.AtBlockID(block1.ID()).RandomSource() + require.NoError(t, err) + }) + }) + + // should be able to get QC and random beacon seed from a block with child(has to be certified) + t.Run("participant-block-processable", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + // create a block to query + block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(flow.EmptyPayload()) + err := state.Extend(context.Background(), block1) + require.NoError(t, err) + + _, err = state.AtBlockID(block1.ID()).QuorumCertificate() + assert.ErrorIs(t, err, storage.ErrNotFound) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), block2) + require.NoError(t, err) + + qc, err := state.AtBlockID(block1.ID()).QuorumCertificate() + require.NoError(t, err) + + // should have view matching block1 view + assert.Equal(t, block1.Header.View, qc.View) + assert.Equal(t, block1.ID(), qc.BlockID) + }) + }) +} + +// test that we can query current/next/previous epochs from a snapshot +func TestSnapshot_EpochQuery(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + epoch1Counter := result.ServiceEvents[0].Event.(*flow.EpochSetup).Counter + epoch2Counter := epoch1Counter + 1 + + epochBuilder := unittest.NewEpochBuilder(t, state) + // build epoch 1 (prepare epoch 2) + epochBuilder. + BuildEpoch(). + CompleteEpoch() + // build epoch 2 (prepare epoch 3) + epochBuilder. + BuildEpoch(). + CompleteEpoch() + + // get heights of each phase in built epochs + epoch1, ok := epochBuilder.EpochHeights(1) + require.True(t, ok) + epoch2, ok := epochBuilder.EpochHeights(2) + require.True(t, ok) + + // we should be able to query the current epoch from any block + t.Run("Current", func(t *testing.T) { + t.Run("epoch 1", func(t *testing.T) { + for _, height := range epoch1.Range() { + counter, err := state.AtHeight(height).Epochs().Current().Counter() + require.NoError(t, err) + assert.Equal(t, epoch1Counter, counter) + } + }) + + t.Run("epoch 2", func(t *testing.T) { + for _, height := range epoch2.Range() { + counter, err := state.AtHeight(height).Epochs().Current().Counter() + require.NoError(t, err) + assert.Equal(t, epoch2Counter, counter) + } + }) + }) + + // we should be unable to query next epoch before it is defined by EpochSetup + // event, afterward we should be able to query next epoch + t.Run("Next", func(t *testing.T) { + t.Run("epoch 1: before next epoch available", func(t *testing.T) { + for _, height := range epoch1.StakingRange() { + _, err := state.AtHeight(height).Epochs().Next().Counter() + assert.Error(t, err) + assert.True(t, errors.Is(err, protocol.ErrNextEpochNotSetup)) + } + }) + + t.Run("epoch 2: after next epoch available", func(t *testing.T) { + for _, height := range append(epoch1.SetupRange(), epoch1.CommittedRange()...) { + counter, err := state.AtHeight(height).Epochs().Next().Counter() + require.NoError(t, err) + assert.Equal(t, epoch2Counter, counter) + } + }) + }) + + // we should get a sentinel error when querying previous epoch from the + // first epoch after the root block, otherwise we should always be able + // to query previous epoch + t.Run("Previous", func(t *testing.T) { + t.Run("epoch 1", func(t *testing.T) { + for _, height := range epoch1.Range() { + _, err := state.AtHeight(height).Epochs().Previous().Counter() + assert.Error(t, err) + assert.True(t, errors.Is(err, protocol.ErrNoPreviousEpoch)) + } + }) + + t.Run("epoch 2", func(t *testing.T) { + for _, height := range epoch2.Range() { + counter, err := state.AtHeight(height).Epochs().Previous().Counter() + require.NoError(t, err) + assert.Equal(t, epoch1Counter, counter) + } + }) + }) + }) +} + +// test that querying the first view of an epoch returns the appropriate value +func TestSnapshot_EpochFirstView(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + head, err := rootSnapshot.Head() + require.NoError(t, err) + result, _, err := rootSnapshot.SealedResult() + require.NoError(t, err) + + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + + epochBuilder := unittest.NewEpochBuilder(t, state) + // build epoch 1 (prepare epoch 2) + epochBuilder. + BuildEpoch(). + CompleteEpoch() + // build epoch 2 (prepare epoch 3) + epochBuilder. + BuildEpoch(). + CompleteEpoch() + + // get heights of each phase in built epochs + epoch1, ok := epochBuilder.EpochHeights(1) + require.True(t, ok) + epoch2, ok := epochBuilder.EpochHeights(2) + require.True(t, ok) + + // figure out the expected first views of the epochs + epoch1FirstView := head.View + epoch2FirstView := result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView + 1 + + // check first view for snapshots within epoch 1, with respect to a + // snapshot in either epoch 1 or epoch 2 (testing Current and Previous) + t.Run("epoch 1", func(t *testing.T) { + + // test w.r.t. epoch 1 snapshot + t.Run("Current", func(t *testing.T) { + for _, height := range epoch1.Range() { + actualFirstView, err := state.AtHeight(height).Epochs().Current().FirstView() + require.NoError(t, err) + assert.Equal(t, epoch1FirstView, actualFirstView) + } + }) + + // test w.r.t. epoch 2 snapshot + t.Run("Previous", func(t *testing.T) { + for _, height := range epoch2.Range() { + actualFirstView, err := state.AtHeight(height).Epochs().Previous().FirstView() + require.NoError(t, err) + assert.Equal(t, epoch1FirstView, actualFirstView) + } + }) + }) + + // check first view for snapshots within epoch 2, with respect to a + // snapshot in either epoch 1 or epoch 2 (testing Next and Current) + t.Run("epoch 2", func(t *testing.T) { + + // test w.r.t. epoch 1 snapshot + t.Run("Next", func(t *testing.T) { + for _, height := range append(epoch1.SetupRange(), epoch1.CommittedRange()...) { + actualFirstView, err := state.AtHeight(height).Epochs().Next().FirstView() + require.NoError(t, err) + assert.Equal(t, epoch2FirstView, actualFirstView) + } + }) + + // test w.r.t. epoch 2 snapshot + t.Run("Current", func(t *testing.T) { + for _, height := range epoch2.Range() { + actualFirstView, err := state.AtHeight(height).Epochs().Current().FirstView() + require.NoError(t, err) + assert.Equal(t, epoch2FirstView, actualFirstView) + } + }) + }) + }) +} + +// TestSnapshot_EpochHeightBoundaries tests querying epoch height boundaries in various conditions. +// - FirstHeight should be queryable as soon as the epoch's first block is finalized, +// otherwise should return protocol.ErrEpochTransitionNotFinalized +// - FinalHeight should be queryable as soon as the next epoch's first block is finalized, +// otherwise should return protocol.ErrEpochTransitionNotFinalized +func TestSnapshot_EpochHeightBoundaries(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + head, err := rootSnapshot.Head() + require.NoError(t, err) + + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + + epochBuilder := unittest.NewEpochBuilder(t, state) + + epoch1FirstHeight := head.Height + t.Run("first epoch - EpochStaking phase", func(t *testing.T) { + // first height of started current epoch should be known + firstHeight, err := state.Final().Epochs().Current().FirstHeight() + require.NoError(t, err) + assert.Equal(t, epoch1FirstHeight, firstHeight) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + }) + + // build first epoch (but don't complete it yet) + epochBuilder.BuildEpoch() + + t.Run("first epoch - EpochCommitted phase", func(t *testing.T) { + // first height of started current epoch should be known + firstHeight, err := state.Final().Epochs().Current().FirstHeight() + require.NoError(t, err) + assert.Equal(t, epoch1FirstHeight, firstHeight) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + // first and final height of not started next epoch should be unknown + _, err = state.Final().Epochs().Next().FirstHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + _, err = state.Final().Epochs().Next().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + }) + + // complete epoch 1 (enter epoch 2) + epochBuilder.CompleteEpoch() + epoch1Heights, ok := epochBuilder.EpochHeights(1) + require.True(t, ok) + epoch1FinalHeight := epoch1Heights.FinalHeight() + epoch2FirstHeight := epoch1FinalHeight + 1 + + t.Run("second epoch - EpochStaking phase", func(t *testing.T) { + // first and final height of completed previous epoch should be known + firstHeight, err := state.Final().Epochs().Previous().FirstHeight() + require.NoError(t, err) + assert.Equal(t, epoch1FirstHeight, firstHeight) + finalHeight, err := state.Final().Epochs().Previous().FinalHeight() + require.NoError(t, err) + assert.Equal(t, epoch1FinalHeight, finalHeight) + + // first height of started current epoch should be known + firstHeight, err = state.Final().Epochs().Current().FirstHeight() + require.NoError(t, err) + assert.Equal(t, epoch2FirstHeight, firstHeight) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + }) + }) +} + +// Test querying identities in different epoch phases. During staking phase we +// should see identities from last epoch and current epoch. After staking phase +// we should see identities from current epoch and next epoch. Identities from +// a non-current epoch should have weight 0. Identities that exist in consecutive +// epochs should be de-duplicated. +func TestSnapshot_CrossEpochIdentities(t *testing.T) { + + // start with 20 identities in epoch 1 + epoch1Identities := unittest.IdentityListFixture(20, unittest.WithAllRoles()) + // 1 identity added at epoch 2 that was not present in epoch 1 + addedAtEpoch2 := unittest.IdentityFixture() + // 1 identity removed in epoch 2 that was present in epoch 1 + removedAtEpoch2 := epoch1Identities[rand.Intn(len(epoch1Identities))] + // epoch 2 has partial overlap with epoch 1 + epoch2Identities := append( + epoch1Identities.Filter(filter.Not(filter.HasNodeID(removedAtEpoch2.NodeID))), + addedAtEpoch2) + // epoch 3 has no overlap with epoch 2 + epoch3Identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) + + rootSnapshot := unittest.RootSnapshotFixture(epoch1Identities) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + + epochBuilder := unittest.NewEpochBuilder(t, state) + // build epoch 1 (prepare epoch 2) + epochBuilder. + UsingSetupOpts(unittest.WithParticipants(epoch2Identities)). + BuildEpoch(). + CompleteEpoch() + // build epoch 2 (prepare epoch 3) + epochBuilder. + UsingSetupOpts(unittest.WithParticipants(epoch3Identities)). + BuildEpoch(). + CompleteEpoch() + + // get heights of each phase in built epochs + epoch1, ok := epochBuilder.EpochHeights(1) + require.True(t, ok) + epoch2, ok := epochBuilder.EpochHeights(2) + require.True(t, ok) + + t.Run("should be able to query at root block", func(t *testing.T) { + root, err := state.Params().FinalizedRoot() + require.NoError(t, err) + snapshot := state.AtHeight(root.Height) + identities, err := snapshot.Identities(filter.Any) + require.NoError(t, err) + + // should have the right number of identities + assert.Equal(t, len(epoch1Identities), len(identities)) + // should have all epoch 1 identities + assert.ElementsMatch(t, epoch1Identities, identities) + }) + + t.Run("should include next epoch after staking phase", func(t *testing.T) { + + // get a snapshot from setup phase and commit phase of epoch 1 + snapshots := []protocol.Snapshot{state.AtHeight(epoch1.Setup), state.AtHeight(epoch1.Committed)} + + for _, snapshot := range snapshots { + phase, err := snapshot.Phase() + require.NoError(t, err) + + t.Run("phase: "+phase.String(), func(t *testing.T) { + identities, err := snapshot.Identities(filter.Any) + require.NoError(t, err) + + // should have the right number of identities + assert.Equal(t, len(epoch1Identities)+1, len(identities)) + // all current epoch identities should match configuration from EpochSetup event + assert.ElementsMatch(t, epoch1Identities, identities.Filter(epoch1Identities.Selector())) + + // should contain single next epoch identity with 0 weight + nextEpochIdentity := identities.Filter(filter.HasNodeID(addedAtEpoch2.NodeID))[0] + assert.Equal(t, uint64(0), nextEpochIdentity.Weight) // should have 0 weight + nextEpochIdentity.Weight = addedAtEpoch2.Weight + assert.Equal(t, addedAtEpoch2, nextEpochIdentity) // should be equal besides weight + }) + } + }) + + t.Run("should include previous epoch in staking phase", func(t *testing.T) { + + // get a snapshot from staking phase of epoch 2 + snapshot := state.AtHeight(epoch2.Staking) + identities, err := snapshot.Identities(filter.Any) + require.NoError(t, err) + + // should have the right number of identities + assert.Equal(t, len(epoch2Identities)+1, len(identities)) + // all current epoch identities should match configuration from EpochSetup event + assert.ElementsMatch(t, epoch2Identities, identities.Filter(epoch2Identities.Selector())) + + // should contain single previous epoch identity with 0 weight + lastEpochIdentity := identities.Filter(filter.HasNodeID(removedAtEpoch2.NodeID))[0] + assert.Equal(t, uint64(0), lastEpochIdentity.Weight) // should have 0 weight + lastEpochIdentity.Weight = removedAtEpoch2.Weight // overwrite weight + assert.Equal(t, removedAtEpoch2, lastEpochIdentity) // should be equal besides weight + }) + + t.Run("should not include previous epoch after staking phase", func(t *testing.T) { + + // get a snapshot from setup phase and commit phase of epoch 2 + snapshots := []protocol.Snapshot{state.AtHeight(epoch2.Setup), state.AtHeight(epoch2.Committed)} + + for _, snapshot := range snapshots { + phase, err := snapshot.Phase() + require.NoError(t, err) + + t.Run("phase: "+phase.String(), func(t *testing.T) { + identities, err := snapshot.Identities(filter.Any) + require.NoError(t, err) + + // should have the right number of identities + assert.Equal(t, len(epoch2Identities)+len(epoch3Identities), len(identities)) + // all current epoch identities should match configuration from EpochSetup event + assert.ElementsMatch(t, epoch2Identities, identities.Filter(epoch2Identities.Selector())) + + // should contain next epoch identities with 0 weight + for _, expected := range epoch3Identities { + actual, exists := identities.ByNodeID(expected.NodeID) + require.True(t, exists) + assert.Equal(t, uint64(0), actual.Weight) // should have 0 weight + actual.Weight = expected.Weight // overwrite weight + assert.Equal(t, expected, actual) // should be equal besides weight + } + }) + } + }) + }) +} + +// test that we can retrieve identities after a spork where the parent ID of the +// root block is non-nil +func TestSnapshot_PostSporkIdentities(t *testing.T) { + expected := unittest.CompleteIdentitySet() + root, result, seal := unittest.BootstrapFixture(expected, func(block *flow.Block) { + block.Header.ParentID = unittest.IdentifierFixture() + }) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) + + rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) + require.NoError(t, err) + + util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.State) { + actual, err := state.Final().Identities(filter.Any) + require.NoError(t, err) + assert.ElementsMatch(t, expected, actual) + }) +} diff --git a/state/protocol/pebble/state.go b/state/protocol/pebble/state.go new file mode 100644 index 00000000000..40973dc05f2 --- /dev/null +++ b/state/protocol/pebble/state.go @@ -0,0 +1,965 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "errors" + "fmt" + "sync/atomic" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + statepkg "github.com/onflow/flow-go/state" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/invalid" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// cachedHeader caches a block header and its ID. +type cachedHeader struct { + id flow.Identifier + header *flow.Header +} + +type State struct { + metrics module.ComplianceMetrics + db *badger.DB + headers storage.Headers + blocks storage.Blocks + qcs storage.QuorumCertificates + results storage.ExecutionResults + seals storage.Seals + epoch struct { + setups storage.EpochSetups + commits storage.EpochCommits + statuses storage.EpochStatuses + } + versionBeacons storage.VersionBeacons + + // rootHeight marks the cutoff of the history this node knows about. We cache it in the state + // because it cannot change over the lifecycle of a protocol state instance. It is frequently + // larger than the height of the root block of the spork, (also cached below as + // `sporkRootBlockHeight`), for instance if the node joined in an epoch after the last spork. + finalizedRootHeight uint64 + // sealedRootHeight returns the root block that is sealed. + sealedRootHeight uint64 + // sporkRootBlockHeight is the height of the root block in the current spork. We cache it in + // the state, because it cannot change over the lifecycle of a protocol state instance. + // Caution: A node that joined in a later epoch past the spork, the node will likely _not_ + // know the spork's root block in full (though it will always know the height). + sporkRootBlockHeight uint64 + // cache the latest finalized and sealed block headers as these are common queries. + // It can be cached because the protocol state is solely responsible for updating these values. + cachedFinal *atomic.Pointer[cachedHeader] + cachedSealed *atomic.Pointer[cachedHeader] +} + +var _ protocol.State = (*State)(nil) + +type BootstrapConfig struct { + // SkipNetworkAddressValidation flags allows skipping all the network address related + // validations not needed for an unstaked node + SkipNetworkAddressValidation bool +} + +func defaultBootstrapConfig() *BootstrapConfig { + return &BootstrapConfig{ + SkipNetworkAddressValidation: false, + } +} + +type BootstrapConfigOptions func(conf *BootstrapConfig) + +func SkipNetworkAddressValidation(conf *BootstrapConfig) { + conf.SkipNetworkAddressValidation = true +} + +func Bootstrap( + metrics module.ComplianceMetrics, + db *badger.DB, + headers storage.Headers, + seals storage.Seals, + results storage.ExecutionResults, + blocks storage.Blocks, + qcs storage.QuorumCertificates, + setups storage.EpochSetups, + commits storage.EpochCommits, + statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, + root protocol.Snapshot, + options ...BootstrapConfigOptions, +) (*State, error) { + + config := defaultBootstrapConfig() + for _, opt := range options { + opt(config) + } + + isBootstrapped, err := IsBootstrapped(db) + if err != nil { + return nil, fmt.Errorf("failed to determine whether database contains bootstrapped state: %w", err) + } + if isBootstrapped { + return nil, fmt.Errorf("expected empty database") + } + + state := newState( + metrics, + db, + headers, + seals, + results, + blocks, + qcs, + setups, + commits, + statuses, + versionBeacons, + ) + + if err := IsValidRootSnapshot(root, !config.SkipNetworkAddressValidation); err != nil { + return nil, fmt.Errorf("cannot bootstrap invalid root snapshot: %w", err) + } + + segment, err := root.SealingSegment() + if err != nil { + return nil, fmt.Errorf("could not get sealing segment: %w", err) + } + + _, rootSeal, err := root.SealedResult() + if err != nil { + return nil, fmt.Errorf("could not get sealed result for sealing segment: %w", err) + } + + err = operation.RetryOnConflictTx(db, transaction.Update, func(tx *transaction.Tx) error { + // sealing segment is in ascending height order, so the tail is the + // oldest ancestor and head is the newest child in the segment + // TAIL <- ... <- HEAD + lastFinalized := segment.Finalized() // the highest block in sealing segment is the last finalized block + lastSealed := segment.Sealed() // the lowest block in sealing segment is the last sealed block + + // 1) bootstrap the sealing segment + // creating sealed root block with the rootResult + // creating finalized root block with lastFinalized + err = state.bootstrapSealingSegment(segment, lastFinalized, rootSeal)(tx) + if err != nil { + return fmt.Errorf("could not bootstrap sealing chain segment blocks: %w", err) + } + + // 2) insert the root quorum certificate into the database + qc, err := root.QuorumCertificate() + if err != nil { + return fmt.Errorf("could not get root qc: %w", err) + } + err = qcs.StoreTx(qc)(tx) + if err != nil { + return fmt.Errorf("could not insert root qc: %w", err) + } + + // 3) initialize the current protocol state height/view pointers + err = transaction.WithTx(state.bootstrapStatePointers(root))(tx) + if err != nil { + return fmt.Errorf("could not bootstrap height/view pointers: %w", err) + } + + // 4) initialize values related to the epoch logic + err = state.bootstrapEpoch(root.Epochs(), segment, !config.SkipNetworkAddressValidation)(tx) + if err != nil { + return fmt.Errorf("could not bootstrap epoch values: %w", err) + } + + // 5) initialize spork params + err = transaction.WithTx(state.bootstrapSporkInfo(root))(tx) + if err != nil { + return fmt.Errorf("could not bootstrap spork info: %w", err) + } + + // 6) set metric values + err = state.updateEpochMetrics(root) + if err != nil { + return fmt.Errorf("could not update epoch metrics: %w", err) + } + state.metrics.BlockSealed(lastSealed) + state.metrics.SealedHeight(lastSealed.Header.Height) + state.metrics.FinalizedHeight(lastFinalized.Header.Height) + for _, block := range segment.Blocks { + state.metrics.BlockFinalized(block) + } + + // 7) initialize version beacon + err = transaction.WithTx(state.boostrapVersionBeacon(root))(tx) + if err != nil { + return fmt.Errorf("could not bootstrap version beacon: %w", err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("bootstrapping failed: %w", err) + } + + // populate the protocol state cache + err = state.populateCache() + if err != nil { + return nil, fmt.Errorf("failed to populate cache: %w", err) + } + + return state, nil +} + +// bootstrapSealingSegment inserts all blocks and associated metadata for the +// protocol state root snapshot to disk. +func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head *flow.Block, rootSeal *flow.Seal) func(tx *transaction.Tx) error { + return func(tx *transaction.Tx) error { + + for _, result := range segment.ExecutionResults { + err := transaction.WithTx(operation.SkipDuplicates(operation.InsertExecutionResult(result)))(tx) + if err != nil { + return fmt.Errorf("could not insert execution result: %w", err) + } + err = transaction.WithTx(operation.IndexExecutionResult(result.BlockID, result.ID()))(tx) + if err != nil { + return fmt.Errorf("could not index execution result: %w", err) + } + } + + // insert the first seal (in case the segment's first block contains no seal) + if segment.FirstSeal != nil { + err := transaction.WithTx(operation.InsertSeal(segment.FirstSeal.ID(), segment.FirstSeal))(tx) + if err != nil { + return fmt.Errorf("could not insert first seal: %w", err) + } + } + + // root seal contains the result ID for the sealed root block. If the sealed root block is + // different from the finalized root block, then it means the node dynamically bootstrapped. + // In that case, we should index the result of the sealed root block so that the EN is able + // to execute the next block. + err := transaction.WithTx(operation.SkipDuplicates(operation.IndexExecutionResult(rootSeal.BlockID, rootSeal.ResultID)))(tx) + if err != nil { + return fmt.Errorf("could not index root result: %w", err) + } + + for _, block := range segment.ExtraBlocks { + blockID := block.ID() + height := block.Header.Height + err := state.blocks.StoreTx(block)(tx) + if err != nil { + return fmt.Errorf("could not insert SealingSegment extra block: %w", err) + } + err = transaction.WithTx(operation.IndexBlockHeight(height, blockID))(tx) + if err != nil { + return fmt.Errorf("could not index SealingSegment extra block (id=%x): %w", blockID, err) + } + err = state.qcs.StoreTx(block.Header.QuorumCertificate())(tx) + if err != nil { + return fmt.Errorf("could not store qc for SealingSegment extra block (id=%x): %w", blockID, err) + } + } + + for i, block := range segment.Blocks { + blockID := block.ID() + height := block.Header.Height + + err := state.blocks.StoreTx(block)(tx) + if err != nil { + return fmt.Errorf("could not insert SealingSegment block: %w", err) + } + err = transaction.WithTx(operation.IndexBlockHeight(height, blockID))(tx) + if err != nil { + return fmt.Errorf("could not index SealingSegment block (id=%x): %w", blockID, err) + } + err = state.qcs.StoreTx(block.Header.QuorumCertificate())(tx) + if err != nil { + return fmt.Errorf("could not store qc for SealingSegment block (id=%x): %w", blockID, err) + } + + // index the latest seal as of this block + latestSealID, ok := segment.LatestSeals[blockID] + if !ok { + return fmt.Errorf("missing latest seal for sealing segment block (id=%s)", blockID) + } + // sanity check: make sure the seal exists + var latestSeal flow.Seal + err = transaction.WithTx(operation.RetrieveSeal(latestSealID, &latestSeal))(tx) + if err != nil { + return fmt.Errorf("could not verify latest seal for block (id=%x) exists: %w", blockID, err) + } + err = transaction.WithTx(operation.IndexLatestSealAtBlock(blockID, latestSealID))(tx) + if err != nil { + return fmt.Errorf("could not index block seal: %w", err) + } + + // for all but the first block in the segment, index the parent->child relationship + if i > 0 { + err = transaction.WithTx(operation.InsertBlockChildren(block.Header.ParentID, []flow.Identifier{blockID}))(tx) + if err != nil { + return fmt.Errorf("could not insert child index for block (id=%x): %w", blockID, err) + } + } + } + + // insert an empty child index for the final block in the segment + err = transaction.WithTx(operation.InsertBlockChildren(head.ID(), nil))(tx) + if err != nil { + return fmt.Errorf("could not insert child index for head block (id=%x): %w", head.ID(), err) + } + + return nil + } +} + +// bootstrapStatePointers instantiates special pointers used to by the protocol +// state to keep track of special block heights and views. +func (state *State) bootstrapStatePointers(root protocol.Snapshot) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + segment, err := root.SealingSegment() + if err != nil { + return fmt.Errorf("could not get sealing segment: %w", err) + } + highest := segment.Finalized() + lowest := segment.Sealed() + // find the finalized seal that seals the lowest block, meaning seal.BlockID == lowest.ID() + seal, err := segment.FinalizedSeal() + if err != nil { + return fmt.Errorf("could not get finalized seal from sealing segment: %w", err) + } + + safetyData := &hotstuff.SafetyData{ + LockedOneChainView: highest.Header.View, + HighestAcknowledgedView: highest.Header.View, + } + + // Per convention, all blocks in the sealing segment must be finalized. Therefore, a QC must + // exist for the `highest` block in the sealing segment. The QC for `highest` should be + // contained in the `root` Snapshot and returned by `root.QuorumCertificate()`. Otherwise, + // the Snapshot is incomplete, because consensus nodes require this QC. To reduce the chance of + // accidental misconfiguration undermining consensus liveness, we do the following sanity checks: + // * `rootQC` should not be nil + // * `rootQC` should be for `highest` block, i.e. its view and blockID should match + rootQC, err := root.QuorumCertificate() + if err != nil { + return fmt.Errorf("could not get root QC: %w", err) + } + if rootQC == nil { + return fmt.Errorf("QC for highest (finalized) block in sealing segment cannot be nil") + } + if rootQC.View != highest.Header.View { + return fmt.Errorf("root QC's view %d does not match the highest block in sealing segment (view %d)", rootQC.View, highest.Header.View) + } + if rootQC.BlockID != highest.Header.ID() { + return fmt.Errorf("root QC is for block %v, which does not match the highest block %v in sealing segment", rootQC.BlockID, highest.Header.ID()) + } + + livenessData := &hotstuff.LivenessData{ + CurrentView: highest.Header.View + 1, + NewestQC: rootQC, + } + + // insert initial views for HotStuff + err = operation.InsertSafetyData(highest.Header.ChainID, safetyData)(tx) + if err != nil { + return fmt.Errorf("could not insert safety data: %w", err) + } + err = operation.InsertLivenessData(highest.Header.ChainID, livenessData)(tx) + if err != nil { + return fmt.Errorf("could not insert liveness data: %w", err) + } + + // insert height pointers + err = operation.InsertRootHeight(highest.Header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert finalized root height: %w", err) + } + // the sealed root height is the lowest block in sealing segment + err = operation.InsertSealedRootHeight(lowest.Header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert sealed root height: %w", err) + } + err = operation.InsertFinalizedHeight(highest.Header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert finalized height: %w", err) + } + err = operation.InsertSealedHeight(lowest.Header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert sealed height: %w", err) + } + err = operation.IndexFinalizedSealByBlockID(seal.BlockID, seal.ID())(tx) + if err != nil { + return fmt.Errorf("could not index sealed block: %w", err) + } + + return nil + } +} + +// bootstrapEpoch bootstraps the protocol state database with information about +// the previous, current, and next epochs as of the root snapshot. +// +// The root snapshot's sealing segment must not straddle any epoch transitions +// or epoch phase transitions. +func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, segment *flow.SealingSegment, verifyNetworkAddress bool) func(*transaction.Tx) error { + return func(tx *transaction.Tx) error { + previous := epochs.Previous() + current := epochs.Current() + next := epochs.Next() + + // build the status as we go + status := new(flow.EpochStatus) + var setups []*flow.EpochSetup + var commits []*flow.EpochCommit + + // insert previous epoch if it exists + _, err := previous.Counter() + if err == nil { + // if there is a previous epoch, both setup and commit events must exist + setup, err := protocol.ToEpochSetup(previous) + if err != nil { + return fmt.Errorf("could not get previous epoch setup event: %w", err) + } + commit, err := protocol.ToEpochCommit(previous) + if err != nil { + return fmt.Errorf("could not get previous epoch commit event: %w", err) + } + + if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + return fmt.Errorf("invalid setup: %w", err) + } + if err := isValidEpochCommit(commit, setup); err != nil { + return fmt.Errorf("invalid commit: %w", err) + } + + err = indexFirstHeight(previous)(tx.DBTxn) + if err != nil { + return fmt.Errorf("could not index epoch first height: %w", err) + } + + setups = append(setups, setup) + commits = append(commits, commit) + status.PreviousEpoch.SetupID = setup.ID() + status.PreviousEpoch.CommitID = commit.ID() + } else if !errors.Is(err, protocol.ErrNoPreviousEpoch) { + return fmt.Errorf("could not retrieve previous epoch: %w", err) + } + + // insert current epoch - both setup and commit events must exist + setup, err := protocol.ToEpochSetup(current) + if err != nil { + return fmt.Errorf("could not get current epoch setup event: %w", err) + } + commit, err := protocol.ToEpochCommit(current) + if err != nil { + return fmt.Errorf("could not get current epoch commit event: %w", err) + } + + if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + return fmt.Errorf("invalid setup: %w", err) + } + if err := isValidEpochCommit(commit, setup); err != nil { + return fmt.Errorf("invalid commit: %w", err) + } + + err = indexFirstHeight(current)(tx.DBTxn) + if err != nil { + return fmt.Errorf("could not index epoch first height: %w", err) + } + + setups = append(setups, setup) + commits = append(commits, commit) + status.CurrentEpoch.SetupID = setup.ID() + status.CurrentEpoch.CommitID = commit.ID() + + // insert next epoch, if it exists + _, err = next.Counter() + if err == nil { + // either only the setup event, or both the setup and commit events must exist + setup, err := protocol.ToEpochSetup(next) + if err != nil { + return fmt.Errorf("could not get next epoch setup event: %w", err) + } + + if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + return fmt.Errorf("invalid setup: %w", err) + } + + setups = append(setups, setup) + status.NextEpoch.SetupID = setup.ID() + commit, err := protocol.ToEpochCommit(next) + if err != nil && !errors.Is(err, protocol.ErrNextEpochNotCommitted) { + return fmt.Errorf("could not get next epoch commit event: %w", err) + } + if err == nil { + if err := isValidEpochCommit(commit, setup); err != nil { + return fmt.Errorf("invalid commit") + } + commits = append(commits, commit) + status.NextEpoch.CommitID = commit.ID() + } + } else if !errors.Is(err, protocol.ErrNextEpochNotSetup) { + return fmt.Errorf("could not get next epoch: %w", err) + } + + // sanity check: ensure epoch status is valid + err = status.Check() + if err != nil { + return fmt.Errorf("bootstrapping resulting in invalid epoch status: %w", err) + } + + // insert all epoch setup/commit service events + for _, setup := range setups { + err = state.epoch.setups.StoreTx(setup)(tx) + if err != nil { + return fmt.Errorf("could not store epoch setup event: %w", err) + } + } + for _, commit := range commits { + err = state.epoch.commits.StoreTx(commit)(tx) + if err != nil { + return fmt.Errorf("could not store epoch commit event: %w", err) + } + } + + // NOTE: as specified in the godoc, this code assumes that each block + // in the sealing segment in within the same phase within the same epoch. + for _, block := range segment.AllBlocks() { + blockID := block.ID() + err = state.epoch.statuses.StoreTx(blockID, status)(tx) + if err != nil { + return fmt.Errorf("could not store epoch status for block (id=%x): %w", blockID, err) + } + } + + return nil + } +} + +// bootstrapSporkInfo bootstraps the protocol state with information about the +// spork which is used to disambiguate Flow networks. +func (state *State) bootstrapSporkInfo(root protocol.Snapshot) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + params := root.Params() + + sporkID, err := params.SporkID() + if err != nil { + return fmt.Errorf("could not get spork ID: %w", err) + } + err = operation.InsertSporkID(sporkID)(tx) + if err != nil { + return fmt.Errorf("could not insert spork ID: %w", err) + } + + sporkRootBlockHeight, err := params.SporkRootBlockHeight() + if err != nil { + return fmt.Errorf("could not get spork root block height: %w", err) + } + err = operation.InsertSporkRootBlockHeight(sporkRootBlockHeight)(tx) + if err != nil { + return fmt.Errorf("could not insert spork root block height: %w", err) + } + + version, err := params.ProtocolVersion() + if err != nil { + return fmt.Errorf("could not get protocol version: %w", err) + } + err = operation.InsertProtocolVersion(version)(tx) + if err != nil { + return fmt.Errorf("could not insert protocol version: %w", err) + } + + threshold, err := params.EpochCommitSafetyThreshold() + if err != nil { + return fmt.Errorf("could not get epoch commit safety threshold: %w", err) + } + err = operation.InsertEpochCommitSafetyThreshold(threshold)(tx) + if err != nil { + return fmt.Errorf("could not insert epoch commit safety threshold: %w", err) + } + + return nil + } +} + +// indexFirstHeight indexes the first height for the epoch, as part of bootstrapping. +// The input epoch must have been started (the first block of the epoch has been finalized). +// No errors are expected during normal operation. +func indexFirstHeight(epoch protocol.Epoch) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + counter, err := epoch.Counter() + if err != nil { + return fmt.Errorf("could not get epoch counter: %w", err) + } + firstHeight, err := epoch.FirstHeight() + if err != nil { + return fmt.Errorf("could not get epoch first height: %w", err) + } + err = operation.InsertEpochFirstHeight(counter, firstHeight)(tx) + if err != nil { + return fmt.Errorf("could not index first height %d for epoch %d: %w", firstHeight, counter, err) + } + return nil + } +} + +func OpenState( + metrics module.ComplianceMetrics, + db *badger.DB, + headers storage.Headers, + seals storage.Seals, + results storage.ExecutionResults, + blocks storage.Blocks, + qcs storage.QuorumCertificates, + setups storage.EpochSetups, + commits storage.EpochCommits, + statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, +) (*State, error) { + isBootstrapped, err := IsBootstrapped(db) + if err != nil { + return nil, fmt.Errorf("failed to determine whether database contains bootstrapped state: %w", err) + } + if !isBootstrapped { + return nil, fmt.Errorf("expected database to contain bootstrapped state") + } + state := newState( + metrics, + db, + headers, + seals, + results, + blocks, + qcs, + setups, + commits, + statuses, + versionBeacons, + ) // populate the protocol state cache + err = state.populateCache() + if err != nil { + return nil, fmt.Errorf("failed to populate cache: %w", err) + } + + // report last finalized and sealed block height + finalSnapshot := state.Final() + head, err := finalSnapshot.Head() + if err != nil { + return nil, fmt.Errorf("unexpected error to get finalized block: %w", err) + } + metrics.FinalizedHeight(head.Height) + + sealed, err := state.Sealed().Head() + if err != nil { + return nil, fmt.Errorf("could not get latest sealed block: %w", err) + } + metrics.SealedHeight(sealed.Height) + + // update all epoch related metrics + err = state.updateEpochMetrics(finalSnapshot) + if err != nil { + return nil, fmt.Errorf("failed to update epoch metrics: %w", err) + } + + return state, nil +} + +func (state *State) Params() protocol.Params { + return Params{state: state} +} + +// Sealed returns a snapshot for the latest sealed block. A latest sealed block +// must always exist, so this function always returns a valid snapshot. +func (state *State) Sealed() protocol.Snapshot { + cached := state.cachedSealed.Load() + if cached == nil { + return invalid.NewSnapshotf("internal inconsistency: no cached sealed header") + } + return NewFinalizedSnapshot(state, cached.id, cached.header) +} + +// Final returns a snapshot for the latest finalized block. A latest finalized +// block must always exist, so this function always returns a valid snapshot. +func (state *State) Final() protocol.Snapshot { + cached := state.cachedFinal.Load() + if cached == nil { + return invalid.NewSnapshotf("internal inconsistency: no cached final header") + } + return NewFinalizedSnapshot(state, cached.id, cached.header) +} + +// AtHeight returns a snapshot for the finalized block at the given height. +// This function may return an invalid.Snapshot with: +// - state.ErrUnknownSnapshotReference: +// -> if no block with the given height has been finalized, even if it is incorporated +// -> if the given height is below the root height +// - exception for critical unexpected storage errors +func (state *State) AtHeight(height uint64) protocol.Snapshot { + // retrieve the block ID for the finalized height + var blockID flow.Identifier + err := state.db.View(operation.LookupBlockHeight(height, &blockID)) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return invalid.NewSnapshotf("unknown finalized height %d: %w", height, statepkg.ErrUnknownSnapshotReference) + } + // critical storage error + return invalid.NewSnapshotf("could not look up block by height: %w", err) + } + return newSnapshotWithIncorporatedReferenceBlock(state, blockID) +} + +// AtBlockID returns a snapshot for the block with the given ID. The block may be +// finalized or un-finalized. +// This function may return an invalid.Snapshot with: +// - state.ErrUnknownSnapshotReference: +// -> if no block with the given ID exists in the state +// - exception for critical unexpected storage errors +func (state *State) AtBlockID(blockID flow.Identifier) protocol.Snapshot { + exists, err := state.headers.Exists(blockID) + if err != nil { + return invalid.NewSnapshotf("could not check existence of reference block: %w", err) + } + if !exists { + return invalid.NewSnapshotf("unknown block %x: %w", blockID, statepkg.ErrUnknownSnapshotReference) + } + return newSnapshotWithIncorporatedReferenceBlock(state, blockID) +} + +// newState initializes a new state backed by the provided a badger database, +// mempools and service components. +// The parameter `expectedBootstrappedState` indicates whether the database +// is expected to contain an already bootstrapped state or not +func newState( + metrics module.ComplianceMetrics, + db *badger.DB, + headers storage.Headers, + seals storage.Seals, + results storage.ExecutionResults, + blocks storage.Blocks, + qcs storage.QuorumCertificates, + setups storage.EpochSetups, + commits storage.EpochCommits, + statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, +) *State { + return &State{ + metrics: metrics, + db: db, + headers: headers, + results: results, + seals: seals, + blocks: blocks, + qcs: qcs, + epoch: struct { + setups storage.EpochSetups + commits storage.EpochCommits + statuses storage.EpochStatuses + }{ + setups: setups, + commits: commits, + statuses: statuses, + }, + versionBeacons: versionBeacons, + cachedFinal: new(atomic.Pointer[cachedHeader]), + cachedSealed: new(atomic.Pointer[cachedHeader]), + } +} + +// IsBootstrapped returns whether the database contains a bootstrapped state +func IsBootstrapped(db *badger.DB) (bool, error) { + var finalized uint64 + err := db.View(operation.RetrieveFinalizedHeight(&finalized)) + if errors.Is(err, storage.ErrNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("retrieving finalized height failed: %w", err) + } + return true, nil +} + +// updateEpochMetrics update the `consensus_compliance_current_epoch_counter` and the +// `consensus_compliance_current_epoch_phase` metric +func (state *State) updateEpochMetrics(snap protocol.Snapshot) error { + + // update epoch counter + counter, err := snap.Epochs().Current().Counter() + if err != nil { + return fmt.Errorf("could not get current epoch counter: %w", err) + } + state.metrics.CurrentEpochCounter(counter) + + // update epoch phase + phase, err := snap.Phase() + if err != nil { + return fmt.Errorf("could not get current epoch counter: %w", err) + } + state.metrics.CurrentEpochPhase(phase) + + // update committed epoch final view + err = state.updateCommittedEpochFinalView(snap) + if err != nil { + return fmt.Errorf("could not update committed epoch final view") + } + + currentEpochFinalView, err := snap.Epochs().Current().FinalView() + if err != nil { + return fmt.Errorf("could not update current epoch final view: %w", err) + } + state.metrics.CurrentEpochFinalView(currentEpochFinalView) + + dkgPhase1FinalView, dkgPhase2FinalView, dkgPhase3FinalView, err := protocol.DKGPhaseViews(snap.Epochs().Current()) + if err != nil { + return fmt.Errorf("could not get dkg phase final view: %w", err) + } + + state.metrics.CurrentDKGPhase1FinalView(dkgPhase1FinalView) + state.metrics.CurrentDKGPhase2FinalView(dkgPhase2FinalView) + state.metrics.CurrentDKGPhase3FinalView(dkgPhase3FinalView) + + // EECC - check whether the epoch emergency fallback flag has been set + // in the database. If so, skip updating any epoch-related metrics. + epochFallbackTriggered, err := state.isEpochEmergencyFallbackTriggered() + if err != nil { + return fmt.Errorf("could not check epoch emergency fallback flag: %w", err) + } + if epochFallbackTriggered { + state.metrics.EpochEmergencyFallbackTriggered() + } + + return nil +} + +// boostrapVersionBeacon bootstraps version beacon, by adding the latest beacon +// to an index, if present. +func (state *State) boostrapVersionBeacon( + snapshot protocol.Snapshot, +) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + versionBeacon, err := snapshot.VersionBeacon() + if err != nil { + return err + } + + if versionBeacon == nil { + return nil + } + + return operation.IndexVersionBeaconByHeight(versionBeacon)(txn) + } +} + +// populateCache is used after opening or bootstrapping the state to populate the cache. +// The cache must be populated before the State receives any queries. +// No errors expected during normal operations. +func (state *State) populateCache() error { + + // cache the initial value for finalized block + err := state.db.View(func(tx *badger.Txn) error { + // root height + err := state.db.View(operation.RetrieveRootHeight(&state.finalizedRootHeight)) + if err != nil { + return fmt.Errorf("could not read root block to populate cache: %w", err) + } + // sealed root height + err = state.db.View(operation.RetrieveSealedRootHeight(&state.sealedRootHeight)) + if err != nil { + return fmt.Errorf("could not read sealed root block to populate cache: %w", err) + } + // spork root block height + err = state.db.View(operation.RetrieveSporkRootBlockHeight(&state.sporkRootBlockHeight)) + if err != nil { + return fmt.Errorf("could not get spork root block height: %w", err) + } + // finalized header + var finalizedHeight uint64 + err = operation.RetrieveFinalizedHeight(&finalizedHeight)(tx) + if err != nil { + return fmt.Errorf("could not lookup finalized height: %w", err) + } + var cachedFinalHeader cachedHeader + err = operation.LookupBlockHeight(finalizedHeight, &cachedFinalHeader.id)(tx) + if err != nil { + return fmt.Errorf("could not lookup finalized id (height=%d): %w", finalizedHeight, err) + } + cachedFinalHeader.header, err = state.headers.ByBlockID(cachedFinalHeader.id) + if err != nil { + return fmt.Errorf("could not get finalized block (id=%x): %w", cachedFinalHeader.id, err) + } + state.cachedFinal.Store(&cachedFinalHeader) + // sealed header + var sealedHeight uint64 + err = operation.RetrieveSealedHeight(&sealedHeight)(tx) + if err != nil { + return fmt.Errorf("could not lookup sealed height: %w", err) + } + var cachedSealedHeader cachedHeader + err = operation.LookupBlockHeight(sealedHeight, &cachedSealedHeader.id)(tx) + if err != nil { + return fmt.Errorf("could not lookup sealed id (height=%d): %w", sealedHeight, err) + } + cachedSealedHeader.header, err = state.headers.ByBlockID(cachedSealedHeader.id) + if err != nil { + return fmt.Errorf("could not get sealed block (id=%x): %w", cachedSealedHeader.id, err) + } + state.cachedSealed.Store(&cachedSealedHeader) + return nil + }) + if err != nil { + return fmt.Errorf("could not cache finalized header: %w", err) + } + + return nil +} + +// updateCommittedEpochFinalView updates the `committed_epoch_final_view` metric +// based on the current epoch phase of the input snapshot. It should be called +// at startup and during transitions between EpochSetup and EpochCommitted phases. +// +// For example, suppose we have epochs N and N+1. +// If we are in epoch N's Staking or Setup Phase, then epoch N's final view should be the value of the metric. +// If we are in epoch N's Committed Phase, then epoch N+1's final view should be the value of the metric. +func (state *State) updateCommittedEpochFinalView(snap protocol.Snapshot) error { + + phase, err := snap.Phase() + if err != nil { + return fmt.Errorf("could not get epoch phase: %w", err) + } + + // update metric based of epoch phase + switch phase { + case flow.EpochPhaseStaking, flow.EpochPhaseSetup: + + // if we are in Staking or Setup phase, then set the metric value to the current epoch's final view + finalView, err := snap.Epochs().Current().FinalView() + if err != nil { + return fmt.Errorf("could not get current epoch final view from snapshot: %w", err) + } + state.metrics.CommittedEpochFinalView(finalView) + case flow.EpochPhaseCommitted: + + // if we are in Committed phase, then set the metric value to the next epoch's final view + finalView, err := snap.Epochs().Next().FinalView() + if err != nil { + return fmt.Errorf("could not get next epoch final view from snapshot: %w", err) + } + state.metrics.CommittedEpochFinalView(finalView) + default: + return fmt.Errorf("invalid phase: %s", phase) + } + + return nil +} + +// isEpochEmergencyFallbackTriggered checks whether epoch fallback has been globally triggered. +// Returns: +// * (true, nil) if epoch fallback is triggered +// * (false, nil) if epoch fallback is not triggered (including if the flag is not set) +// * (false, err) if an unexpected error occurs +func (state *State) isEpochEmergencyFallbackTriggered() (bool, error) { + var triggered bool + err := state.db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) + return triggered, err +} diff --git a/state/protocol/pebble/state_test.go b/state/protocol/pebble/state_test.go new file mode 100644 index 00000000000..c6bcc59854f --- /dev/null +++ b/state/protocol/pebble/state_test.go @@ -0,0 +1,642 @@ +package badger_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + testmock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/state/protocol" + bprotocol "github.com/onflow/flow-go/state/protocol/badger" + "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/util" + protoutil "github.com/onflow/flow-go/state/protocol/util" + storagebadger "github.com/onflow/flow-go/storage/badger" + storutil "github.com/onflow/flow-go/storage/util" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestBootstrapAndOpen verifies after bootstrapping with a root snapshot +// we should be able to open it and got the same state. +func TestBootstrapAndOpen(t *testing.T) { + + // create a state root and bootstrap the protocol state with it + participants := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(participants, func(block *flow.Block) { + block.Header.ParentID = unittest.IdentifierFixture() + }) + + protoutil.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, _ *bprotocol.State) { + + // expect the final view metric to be set to current epoch's final view + epoch := rootSnapshot.Epochs().Current() + finalView, err := epoch.FinalView() + require.NoError(t, err) + counter, err := epoch.Counter() + require.NoError(t, err) + phase, err := rootSnapshot.Phase() + require.NoError(t, err) + + complianceMetrics := new(mock.ComplianceMetrics) + complianceMetrics.On("CommittedEpochFinalView", finalView).Once() + complianceMetrics.On("CurrentEpochCounter", counter).Once() + complianceMetrics.On("CurrentEpochPhase", phase).Once() + complianceMetrics.On("CurrentEpochFinalView", finalView).Once() + complianceMetrics.On("FinalizedHeight", testmock.Anything).Once() + complianceMetrics.On("SealedHeight", testmock.Anything).Once() + + dkgPhase1FinalView, dkgPhase2FinalView, dkgPhase3FinalView, err := protocol.DKGPhaseViews(epoch) + require.NoError(t, err) + complianceMetrics.On("CurrentDKGPhase1FinalView", dkgPhase1FinalView).Once() + complianceMetrics.On("CurrentDKGPhase2FinalView", dkgPhase2FinalView).Once() + complianceMetrics.On("CurrentDKGPhase3FinalView", dkgPhase3FinalView).Once() + + noopMetrics := new(metrics.NoopCollector) + all := storagebadger.InitAll(noopMetrics, db) + // protocol state has been bootstrapped, now open a protocol state with the database + state, err := bprotocol.OpenState( + complianceMetrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + ) + require.NoError(t, err) + + complianceMetrics.AssertExpectations(t) + + unittest.AssertSnapshotsEqual(t, rootSnapshot, state.Final()) + + vb, err := state.Final().VersionBeacon() + require.NoError(t, err) + require.Nil(t, vb) + }) +} + +// TestBootstrapAndOpen_EpochCommitted verifies after bootstrapping with a +// root snapshot from EpochCommitted phase we should be able to open it and +// got the same state. +func TestBootstrapAndOpen_EpochCommitted(t *testing.T) { + + // create a state root and bootstrap the protocol state with it + participants := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(participants, func(block *flow.Block) { + block.Header.ParentID = unittest.IdentifierFixture() + }) + rootBlock, err := rootSnapshot.Head() + require.NoError(t, err) + + // build an epoch on the root state and return a snapshot from the committed phase + committedPhaseSnapshot := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + unittest.NewEpochBuilder(t, state).BuildEpoch().CompleteEpoch() + + // find the point where we transition to the epoch committed phase + for height := rootBlock.Height + 1; ; height++ { + phase, err := state.AtHeight(height).Phase() + require.NoError(t, err) + if phase == flow.EpochPhaseCommitted { + return state.AtHeight(height) + } + } + }) + + protoutil.RunWithBootstrapState(t, committedPhaseSnapshot, func(db *badger.DB, _ *bprotocol.State) { + + complianceMetrics := new(mock.ComplianceMetrics) + + // expect the final view metric to be set to next epoch's final view + finalView, err := committedPhaseSnapshot.Epochs().Next().FinalView() + require.NoError(t, err) + complianceMetrics.On("CommittedEpochFinalView", finalView).Once() + + // expect counter to be set to current epochs counter + counter, err := committedPhaseSnapshot.Epochs().Current().Counter() + require.NoError(t, err) + complianceMetrics.On("CurrentEpochCounter", counter).Once() + + // expect epoch phase to be set to current phase + phase, err := committedPhaseSnapshot.Phase() + require.NoError(t, err) + complianceMetrics.On("CurrentEpochPhase", phase).Once() + + currentEpochFinalView, err := committedPhaseSnapshot.Epochs().Current().FinalView() + require.NoError(t, err) + complianceMetrics.On("CurrentEpochFinalView", currentEpochFinalView).Once() + + dkgPhase1FinalView, dkgPhase2FinalView, dkgPhase3FinalView, err := protocol.DKGPhaseViews(committedPhaseSnapshot.Epochs().Current()) + require.NoError(t, err) + complianceMetrics.On("CurrentDKGPhase1FinalView", dkgPhase1FinalView).Once() + complianceMetrics.On("CurrentDKGPhase2FinalView", dkgPhase2FinalView).Once() + complianceMetrics.On("CurrentDKGPhase3FinalView", dkgPhase3FinalView).Once() + complianceMetrics.On("FinalizedHeight", testmock.Anything).Once() + complianceMetrics.On("SealedHeight", testmock.Anything).Once() + + noopMetrics := new(metrics.NoopCollector) + all := storagebadger.InitAll(noopMetrics, db) + state, err := bprotocol.OpenState( + complianceMetrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + ) + require.NoError(t, err) + + // assert update final view was called + complianceMetrics.AssertExpectations(t) + + unittest.AssertSnapshotsEqual(t, committedPhaseSnapshot, state.Final()) + }) +} + +// TestBootstrap_EpochHeightBoundaries tests that epoch height indexes are indexed +// when they are available in the input snapshot. +func TestBootstrap_EpochHeightBoundaries(t *testing.T) { + t.Parallel() + // start with a regular post-spork root snapshot + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + epoch1FirstHeight := rootSnapshot.Encodable().Head.Height + + t.Run("root snapshot", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // first height of started current epoch should be known + firstHeight, err := state.Final().Epochs().Current().FirstHeight() + require.NoError(t, err) + assert.Equal(t, epoch1FirstHeight, firstHeight) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + }) + }) + + t.Run("with next epoch", func(t *testing.T) { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, state) + builder.BuildEpoch().CompleteEpoch() + heights, ok := builder.EpochHeights(1) + require.True(t, ok) + return state.AtHeight(heights.Committed) + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + // first height of started current epoch should be known + firstHeight, err := state.Final().Epochs().Current().FirstHeight() + assert.Equal(t, epoch1FirstHeight, firstHeight) + require.NoError(t, err) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + // first and final height of not started next epoch should be unknown + _, err = state.Final().Epochs().Next().FirstHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + _, err = state.Final().Epochs().Next().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + }) + }) + t.Run("with previous epoch", func(t *testing.T) { + var epoch1FinalHeight uint64 + var epoch2FirstHeight uint64 + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, state) + builder. + BuildEpoch().CompleteEpoch(). // build epoch 2 + BuildEpoch() // build epoch 3 + heights, ok := builder.EpochHeights(2) + epoch2FirstHeight = heights.FirstHeight() + epoch1FinalHeight = epoch2FirstHeight - 1 + require.True(t, ok) + // return snapshot from within epoch 2 (middle epoch) + return state.AtHeight(heights.Setup) + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + // first height of started current epoch should be known + firstHeight, err := state.Final().Epochs().Current().FirstHeight() + assert.Equal(t, epoch2FirstHeight, firstHeight) + require.NoError(t, err) + // final height of not completed current epoch should be unknown + _, err = state.Final().Epochs().Current().FinalHeight() + assert.ErrorIs(t, err, protocol.ErrEpochTransitionNotFinalized) + // first and final height of completed previous epoch should be known + firstHeight, err = state.Final().Epochs().Previous().FirstHeight() + require.NoError(t, err) + assert.Equal(t, firstHeight, epoch1FirstHeight) + finalHeight, err := state.Final().Epochs().Previous().FinalHeight() + require.NoError(t, err) + assert.Equal(t, finalHeight, epoch1FinalHeight) + }) + }) +} + +// TestBootstrapNonRoot tests bootstrapping the protocol state from arbitrary states. +// +// NOTE: for all these cases, we build a final child block (CHILD). This is +// needed otherwise the parent block would not have a valid QC, since the QC +// is stored in the child. +func TestBootstrapNonRoot(t *testing.T) { + t.Parallel() + // start with a regular post-spork root snapshot + participants := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(participants) + rootBlock, err := rootSnapshot.Head() + require.NoError(t, err) + + // should be able to bootstrap from snapshot after sealing a non-root block + // ROOT <- B1 <- B2(R1) <- B3(S1) <- CHILD + t.Run("with sealed block", func(t *testing.T) { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + block1 := unittest.BlockWithParentFixture(rootBlock) + buildFinalizedBlock(t, state, block1) + + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block3) + + child := unittest.BlockWithParentFixture(block3.Header) + buildBlock(t, state, child) + + return state.AtBlockID(block3.ID()) + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + unittest.AssertSnapshotsEqual(t, after, state.Final()) + // should be able to read all QCs + segment, err := state.Final().SealingSegment() + require.NoError(t, err) + for _, block := range segment.Blocks { + snapshot := state.AtBlockID(block.ID()) + _, err := snapshot.QuorumCertificate() + require.NoError(t, err) + _, err = snapshot.RandomSource() + require.NoError(t, err) + } + }) + }) + + t.Run("with setup next epoch", func(t *testing.T) { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + unittest.NewEpochBuilder(t, state).BuildEpoch() + + // find the point where we transition to the epoch setup phase + for height := rootBlock.Height + 1; ; height++ { + phase, err := state.AtHeight(height).Phase() + require.NoError(t, err) + if phase == flow.EpochPhaseSetup { + return state.AtHeight(height) + } + } + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + unittest.AssertSnapshotsEqual(t, after, state.Final()) + }) + }) + + t.Run("with committed next epoch", func(t *testing.T) { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + unittest.NewEpochBuilder(t, state).BuildEpoch().CompleteEpoch() + + // find the point where we transition to the epoch committed phase + for height := rootBlock.Height + 1; ; height++ { + phase, err := state.AtHeight(height).Phase() + require.NoError(t, err) + if phase == flow.EpochPhaseCommitted { + return state.AtHeight(height) + } + } + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + unittest.AssertSnapshotsEqual(t, after, state.Final()) + }) + }) + + t.Run("with previous and next epoch", func(t *testing.T) { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + unittest.NewEpochBuilder(t, state). + BuildEpoch().CompleteEpoch(). // build epoch 2 + BuildEpoch() // build epoch 3 + + // find a snapshot from epoch setup phase in epoch 2 + epoch1Counter, err := rootSnapshot.Epochs().Current().Counter() + require.NoError(t, err) + for height := rootBlock.Height + 1; ; height++ { + snap := state.AtHeight(height) + counter, err := snap.Epochs().Current().Counter() + require.NoError(t, err) + phase, err := snap.Phase() + require.NoError(t, err) + if phase == flow.EpochPhaseSetup && counter == epoch1Counter+1 { + return snap + } + } + }) + + bootstrap(t, after, func(state *bprotocol.State, err error) { + require.NoError(t, err) + unittest.AssertSnapshotsEqual(t, after, state.Final()) + }) + }) +} + +func TestBootstrap_InvalidIdentities(t *testing.T) { + t.Run("duplicate node ID", func(t *testing.T) { + participants := unittest.CompleteIdentitySet() + dupeIDIdentity := unittest.IdentityFixture(unittest.WithNodeID(participants[0].NodeID)) + participants = append(participants, dupeIDIdentity) + + root := unittest.RootSnapshotFixture(participants) + bootstrap(t, root, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + + t.Run("zero weight", func(t *testing.T) { + zeroWeightIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification), unittest.WithWeight(0)) + participants := unittest.CompleteIdentitySet(zeroWeightIdentity) + root := unittest.RootSnapshotFixture(participants) + bootstrap(t, root, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + + t.Run("missing role", func(t *testing.T) { + requiredRoles := []flow.Role{ + flow.RoleConsensus, + flow.RoleExecution, + flow.RoleVerification, + } + + for _, role := range requiredRoles { + t.Run(fmt.Sprintf("no %s nodes", role), func(t *testing.T) { + participants := unittest.IdentityListFixture(5, unittest.WithAllRolesExcept(role)) + root := unittest.RootSnapshotFixture(participants) + bootstrap(t, root, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + } + }) + + t.Run("duplicate address", func(t *testing.T) { + participants := unittest.CompleteIdentitySet() + dupeAddressIdentity := unittest.IdentityFixture(unittest.WithAddress(participants[0].Address)) + participants = append(participants, dupeAddressIdentity) + + root := unittest.RootSnapshotFixture(participants) + bootstrap(t, root, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + + t.Run("non-canonical ordering", func(t *testing.T) { + participants := unittest.IdentityListFixture(20, unittest.WithAllRoles()) + + root := unittest.RootSnapshotFixture(participants) + // randomly shuffle the identities so they are not canonically ordered + encodable := root.Encodable() + var err error + encodable.Identities, err = participants.Shuffle() + require.NoError(t, err) + root = inmem.SnapshotFromEncodable(encodable) + bootstrap(t, root, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) +} + +func TestBootstrap_DisconnectedSealingSegment(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + // add an un-connected tail block to the sealing segment + tail := unittest.BlockFixture() + encodable.SealingSegment.Blocks = append([]*flow.Block{&tail}, encodable.SealingSegment.Blocks...) + rootSnapshot = inmem.SnapshotFromEncodable(encodable) + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) +} + +func TestBootstrap_SealingSegmentMissingSeal(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + // we are missing the required first seal + encodable.SealingSegment.FirstSeal = nil + rootSnapshot = inmem.SnapshotFromEncodable(encodable) + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) +} + +func TestBootstrap_SealingSegmentMissingResult(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + // we are missing the result referenced by the root seal + encodable.SealingSegment.ExecutionResults = nil + rootSnapshot = inmem.SnapshotFromEncodable(encodable) + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) +} + +func TestBootstrap_InvalidQuorumCertificate(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + encodable.QuorumCertificate.BlockID = unittest.IdentifierFixture() + rootSnapshot = inmem.SnapshotFromEncodable(encodable) + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) +} + +func TestBootstrap_SealMismatch(t *testing.T) { + t.Run("seal doesn't match tail block", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + encodable.LatestSeal.BlockID = unittest.IdentifierFixture() + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + + t.Run("result doesn't match tail block", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + encodable.LatestResult.BlockID = unittest.IdentifierFixture() + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) + + t.Run("seal doesn't match result", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + // convert to encodable to easily modify snapshot + encodable := rootSnapshot.Encodable() + encodable.LatestSeal.ResultID = unittest.IdentifierFixture() + + bootstrap(t, rootSnapshot, func(state *bprotocol.State, err error) { + assert.Error(t, err) + }) + }) +} + +// bootstraps protocol state with the given snapshot and invokes the callback +// with the result of the constructor +func bootstrap(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.State, error)) { + metrics := metrics.NewNoopCollector() + dir := unittest.TempDir(t) + defer os.RemoveAll(dir) + db := unittest.BadgerDB(t, dir) + defer db.Close() + all := storutil.StorageLayer(t, db) + state, err := bprotocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) + f(state, err) +} + +// snapshotAfter bootstraps the protocol state from the root snapshot, applies +// the state-changing function f, clears the on-disk state, and returns a +// memory-backed snapshot corresponding to that returned by f. +// +// This is used for generating valid snapshots to use when testing bootstrapping +// from non-root states. +func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.FollowerState) protocol.Snapshot) protocol.Snapshot { + var after protocol.Snapshot + protoutil.RunWithFollowerProtocolState(t, rootSnapshot, func(_ *badger.DB, state *bprotocol.FollowerState) { + snap := f(state) + var err error + after, err = inmem.FromSnapshot(snap) + require.NoError(t, err) + }) + return after +} + +// buildBlock extends the protocol state by the given block +func buildBlock(t *testing.T, state protocol.FollowerState, block *flow.Block) { + require.NoError(t, state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))) +} + +// buildFinalizedBlock extends the protocol state by the given block and marks the block as finalized +func buildFinalizedBlock(t *testing.T, state protocol.FollowerState, block *flow.Block) { + require.NoError(t, state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))) + require.NoError(t, state.Finalize(context.Background(), block.ID())) +} + +// assertSealingSegmentBlocksQueryable bootstraps the state with the given +// snapshot, then verifies that all sealing segment blocks are queryable. +func assertSealingSegmentBlocksQueryableAfterBootstrap(t *testing.T, snapshot protocol.Snapshot) { + bootstrap(t, snapshot, func(state *bprotocol.State, err error) { + require.NoError(t, err) + + segment, err := state.Final().SealingSegment() + require.NoError(t, err) + + rootBlock, err := state.Params().FinalizedRoot() + require.NoError(t, err) + + // root block should be the highest block from the sealing segment + assert.Equal(t, segment.Highest().Header, rootBlock) + + // for each block in the sealing segment we should be able to query: + // * Head + // * SealedResult + // * Commit + for _, block := range segment.Blocks { + blockID := block.ID() + snap := state.AtBlockID(blockID) + header, err := snap.Head() + assert.NoError(t, err) + assert.Equal(t, blockID, header.ID()) + _, seal, err := snap.SealedResult() + assert.NoError(t, err) + assert.Equal(t, segment.LatestSeals[blockID], seal.ID()) + commit, err := snap.Commit() + assert.NoError(t, err) + assert.Equal(t, seal.FinalState, commit) + } + // for all blocks but the head, we should be unable to query SealingSegment: + for _, block := range segment.Blocks[:len(segment.Blocks)-1] { + snap := state.AtBlockID(block.ID()) + _, err := snap.SealingSegment() + assert.ErrorIs(t, err, protocol.ErrSealingSegmentBelowRootBlock) + } + }) +} + +// BenchmarkFinal benchmarks retrieving the latest finalized block from storage. +func BenchmarkFinal(b *testing.B) { + util.RunWithBootstrapState(b, unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()), func(db *badger.DB, state *bprotocol.State) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + header, err := state.Final().Head() + assert.NoError(b, err) + assert.NotNil(b, header) + } + }) +} + +// BenchmarkFinal benchmarks retrieving the block by height from storage. +func BenchmarkByHeight(b *testing.B) { + util.RunWithBootstrapState(b, unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()), func(db *badger.DB, state *bprotocol.State) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + header, err := state.AtHeight(0).Head() + assert.NoError(b, err) + assert.NotNil(b, header) + } + }) +} diff --git a/state/protocol/pebble/validity.go b/state/protocol/pebble/validity.go new file mode 100644 index 00000000000..acece515f64 --- /dev/null +++ b/state/protocol/pebble/validity.go @@ -0,0 +1,448 @@ +package badger + +import ( + "fmt" + + "github.com/onflow/flow-go/consensus/hotstuff/committees" + "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/consensus/hotstuff/validator" + "github.com/onflow/flow-go/consensus/hotstuff/verification" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/factory" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" +) + +// isValidExtendingEpochSetup checks whether an epoch setup service being +// added to the state is valid. In addition to intrinsic validity, we also +// check that it is valid w.r.t. the previous epoch setup event, and the +// current epoch status. +// Assumes all inputs besides extendingSetup are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status +func isValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { + // We should only have a single epoch setup event per epoch. + if status.NextEpoch.SetupID != flow.ZeroID { + // true iff EpochSetup event for NEXT epoch was already included before + return protocol.NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) + } + + // The setup event should have the counter increased by one. + if extendingSetup.Counter != activeSetup.Counter+1 { + return protocol.NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) + } + + // The first view needs to be exactly one greater than the current epoch final view + if extendingSetup.FirstView != activeSetup.FinalView+1 { + return protocol.NewInvalidServiceEventErrorf( + "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", + extendingSetup.FirstView, + activeSetup.FinalView, + ) + } + + // Finally, the epoch setup event must contain all necessary information. + err := verifyEpochSetup(extendingSetup, true) + if err != nil { + return protocol.NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) + } + + return nil +} + +// verifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. +// The boolean parameter `verifyNetworkAddress` controls, whether we want to permit +// nodes to share a networking address. +// This is a side-effect-free function. Any error return indicates that the +// EpochSetup event is not compliant with protocol rules. +func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { + // STEP 1: general sanity checks + // the seed needs to be at least minimum length + if len(setup.RandomSource) != flow.EpochSetupRandomSourceLength { + return fmt.Errorf("seed has incorrect length (%d != %d)", len(setup.RandomSource), flow.EpochSetupRandomSourceLength) + } + + // STEP 2: sanity checks of all nodes listed as participants + // there should be no duplicate node IDs + identLookup := make(map[flow.Identifier]struct{}) + for _, participant := range setup.Participants { + _, ok := identLookup[participant.NodeID] + if ok { + return fmt.Errorf("duplicate node identifier (%x)", participant.NodeID) + } + identLookup[participant.NodeID] = struct{}{} + } + + if verifyNetworkAddress { + // there should be no duplicate node addresses + addrLookup := make(map[string]struct{}) + for _, participant := range setup.Participants { + _, ok := addrLookup[participant.Address] + if ok { + return fmt.Errorf("duplicate node address (%x)", participant.Address) + } + addrLookup[participant.Address] = struct{}{} + } + } + + // the participants must be listed in canonical order + if !flow.IsIdentityListCanonical(setup.Participants) { + return fmt.Errorf("participants are not canonically ordered") + } + + // STEP 3: sanity checks for individual roles + // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake + // in communication but not in respective node functions + activeParticipants := setup.Participants.Filter(filter.HasWeight(true)) + + // we need at least one node of each role + roles := make(map[flow.Role]uint) + for _, participant := range activeParticipants { + roles[participant.Role]++ + } + if roles[flow.RoleConsensus] < 1 { + return fmt.Errorf("need at least one consensus node") + } + if roles[flow.RoleCollection] < 1 { + return fmt.Errorf("need at least one collection node") + } + if roles[flow.RoleExecution] < 1 { + return fmt.Errorf("need at least one execution node") + } + if roles[flow.RoleVerification] < 1 { + return fmt.Errorf("need at least one verification node") + } + + // first view must be before final view + if setup.FirstView >= setup.FinalView { + return fmt.Errorf("first view (%d) must be before final view (%d)", setup.FirstView, setup.FinalView) + } + + // we need at least one collection cluster + if len(setup.Assignments) == 0 { + return fmt.Errorf("need at least one collection cluster") + } + + // the collection cluster assignments need to be valid + _, err := factory.NewClusterList(setup.Assignments, activeParticipants.Filter(filter.HasRole(flow.RoleCollection))) + if err != nil { + return fmt.Errorf("invalid cluster assignments: %w", err) + } + + return nil +} + +// isValidExtendingEpochCommit checks whether an epoch commit service being +// added to the state is valid. In addition to intrinsic validity, we also +// check that it is valid w.r.t. the previous epoch setup event, and the +// current epoch status. +// Assumes all inputs besides extendingCommit are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status +func isValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { + + // We should only have a single epoch commit event per epoch. + if status.NextEpoch.CommitID != flow.ZeroID { + // true iff EpochCommit event for NEXT epoch was already included before + return protocol.NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) + } + + // The epoch setup event needs to happen before the commit. + if status.NextEpoch.SetupID == flow.ZeroID { + return protocol.NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") + } + + // The commit event should have the counter increased by one. + if extendingCommit.Counter != activeSetup.Counter+1 { + return protocol.NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) + } + + err := isValidEpochCommit(extendingCommit, extendingSetup) + if err != nil { + return protocol.NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) + } + + return nil +} + +// isValidEpochCommit checks whether an epoch commit service event is intrinsically valid. +// Assumes the input flow.EpochSetup event has already been validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the EpochCommit is invalid +func isValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error { + + if len(setup.Assignments) != len(commit.ClusterQCs) { + return protocol.NewInvalidServiceEventErrorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) + } + + if commit.Counter != setup.Counter { + return protocol.NewInvalidServiceEventErrorf("inconsistent epoch counter between commit (%d) and setup (%d) events in same epoch", commit.Counter, setup.Counter) + } + + // make sure we have a valid DKG public key + if commit.DKGGroupKey == nil { + return protocol.NewInvalidServiceEventErrorf("missing DKG public group key") + } + + participants := setup.Participants.Filter(filter.IsValidDKGParticipant) + if len(participants) != len(commit.DKGParticipantKeys) { + return protocol.NewInvalidServiceEventErrorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) + } + + return nil +} + +// IsValidRootSnapshot checks internal consistency of root state snapshot +// if verifyResultID allows/disallows Result ID verification +func IsValidRootSnapshot(snap protocol.Snapshot, verifyResultID bool) error { + + segment, err := snap.SealingSegment() + if err != nil { + return fmt.Errorf("could not get sealing segment: %w", err) + } + result, seal, err := snap.SealedResult() + if err != nil { + return fmt.Errorf("could not latest sealed result: %w", err) + } + + err = segment.Validate() + if err != nil { + return fmt.Errorf("invalid root sealing segment: %w", err) + } + + highest := segment.Highest() // reference block of the snapshot + lowest := segment.Sealed() // last sealed block + highestID := highest.ID() + lowestID := lowest.ID() + + if result.BlockID != lowestID { + return fmt.Errorf("root execution result for wrong block (%x != %x)", result.BlockID, lowest.ID()) + } + + if seal.BlockID != lowestID { + return fmt.Errorf("root block seal for wrong block (%x != %x)", seal.BlockID, lowest.ID()) + } + + if verifyResultID { + if seal.ResultID != result.ID() { + return fmt.Errorf("root block seal for wrong execution result (%x != %x)", seal.ResultID, result.ID()) + } + } + + // identities must be canonically ordered + identities, err := snap.Identities(filter.Any) + if err != nil { + return fmt.Errorf("could not get identities for root snapshot: %w", err) + } + if !flow.IsIdentityListCanonical(identities) { + return fmt.Errorf("identities are not canonically ordered") + } + + // root qc must be for reference block of snapshot + qc, err := snap.QuorumCertificate() + if err != nil { + return fmt.Errorf("could not get qc for root snapshot: %w", err) + } + if qc.BlockID != highestID { + return fmt.Errorf("qc is for wrong block (got: %x, expected: %x)", qc.BlockID, highestID) + } + + firstView, err := snap.Epochs().Current().FirstView() + if err != nil { + return fmt.Errorf("could not get first view: %w", err) + } + finalView, err := snap.Epochs().Current().FinalView() + if err != nil { + return fmt.Errorf("could not get final view: %w", err) + } + + // the segment must be fully within the current epoch + if firstView > lowest.Header.View { + return fmt.Errorf("lowest block of sealing segment has lower view than first view of epoch") + } + if highest.Header.View >= finalView { + return fmt.Errorf("final view of epoch less than first block view") + } + + err = validateVersionBeacon(snap) + if err != nil { + return err + } + + return nil +} + +// IsValidRootSnapshotQCs checks internal consistency of QCs that are included in the root state snapshot +// It verifies QCs for main consensus and for each collection cluster. +func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { + // validate main consensus QC + err := validateRootQC(snap) + if err != nil { + return fmt.Errorf("invalid root QC: %w", err) + } + + // validate each collection cluster separately + curEpoch := snap.Epochs().Current() + clusters, err := curEpoch.Clustering() + if err != nil { + return fmt.Errorf("could not get clustering for root snapshot: %w", err) + } + for clusterIndex := range clusters { + cluster, err := curEpoch.Cluster(uint(clusterIndex)) + if err != nil { + return fmt.Errorf("could not get cluster %d for root snapshot: %w", clusterIndex, err) + } + err = validateClusterQC(cluster) + if err != nil { + return fmt.Errorf("invalid cluster qc %d: %w", clusterIndex, err) + } + } + return nil +} + +// validateRootQC performs validation of root QC +// Returns nil on success +func validateRootQC(snap protocol.Snapshot) error { + identities, err := snap.Identities(filter.IsVotingConsensusCommitteeMember) + if err != nil { + return fmt.Errorf("could not get root snapshot identities: %w", err) + } + + rootQC, err := snap.QuorumCertificate() + if err != nil { + return fmt.Errorf("could not get root QC: %w", err) + } + + dkg, err := snap.Epochs().Current().DKG() + if err != nil { + return fmt.Errorf("could not get DKG for root snapshot: %w", err) + } + + committee, err := committees.NewStaticCommitteeWithDKG(identities, flow.Identifier{}, dkg) + if err != nil { + return fmt.Errorf("could not create static committee: %w", err) + } + verifier := verification.NewCombinedVerifier(committee, signature.NewConsensusSigDataPacker(committee)) + hotstuffValidator := validator.New(committee, verifier) + err = hotstuffValidator.ValidateQC(rootQC) + if err != nil { + return fmt.Errorf("could not validate root qc: %w", err) + } + return nil +} + +// validateClusterQC performs QC validation of single collection cluster +// Returns nil on success +func validateClusterQC(cluster protocol.Cluster) error { + committee, err := committees.NewStaticCommittee(cluster.Members(), flow.Identifier{}, nil, nil) + if err != nil { + return fmt.Errorf("could not create static committee: %w", err) + } + verifier := verification.NewStakingVerifier() + hotstuffValidator := validator.New(committee, verifier) + err = hotstuffValidator.ValidateQC(cluster.RootQC()) + if err != nil { + return fmt.Errorf("could not validate root qc: %w", err) + } + return nil +} + +// validateVersionBeacon returns an InvalidServiceEventError if the snapshot +// version beacon is invalid +func validateVersionBeacon(snap protocol.Snapshot) error { + errf := func(msg string, args ...any) error { + return protocol.NewInvalidServiceEventErrorf(msg, args) + } + + versionBeacon, err := snap.VersionBeacon() + if err != nil { + return errf("could not get version beacon: %w", err) + } + + if versionBeacon == nil { + return nil + } + + head, err := snap.Head() + if err != nil { + return errf("could not get snapshot head: %w", err) + } + + // version beacon must be included in a past block to be effective + if versionBeacon.SealHeight > head.Height { + return errf("version table height higher than highest height") + } + + err = versionBeacon.Validate() + if err != nil { + return errf("version beacon is invalid: %w", err) + } + + return nil +} + +// ValidRootSnapshotContainsEntityExpiryRange performs a sanity check to make sure the +// root snapshot has enough history to encompass at least one full entity expiry window. +// Entities (in particular transactions and collections) may reference a block within +// the past `flow.DefaultTransactionExpiry` blocks, so a new node must begin with at least +// this many blocks worth of history leading up to the snapshot's root block. +// +// Currently, Access Nodes and Consensus Nodes require root snapshots passing this validator function. +// +// - Consensus Nodes because they process guarantees referencing past blocks +// - Access Nodes because they index transactions referencing past blocks +// +// One of the following conditions must be satisfied to pass this validation: +// 1. This is a snapshot build from a first block of spork +// -> there is no earlier history which transactions/collections could reference +// 2. This snapshot sealing segment contains at least one expiry window of blocks +// -> all possible reference blocks in future transactions/collections will be within the initial history. +// 3. This snapshot sealing segment includes the spork root block +// -> there is no earlier history which transactions/collections could reference +func ValidRootSnapshotContainsEntityExpiryRange(snapshot protocol.Snapshot) error { + isSporkRootSnapshot, err := protocol.IsSporkRootSnapshot(snapshot) + if err != nil { + return fmt.Errorf("could not check if root snapshot is a spork root snapshot: %w", err) + } + // Condition 1 satisfied + if isSporkRootSnapshot { + return nil + } + + head, err := snapshot.Head() + if err != nil { + return fmt.Errorf("could not query root snapshot head: %w", err) + } + + sporkRootBlockHeight, err := snapshot.Params().SporkRootBlockHeight() + if err != nil { + return fmt.Errorf("could not query spork root block height: %w", err) + } + + sealingSegment, err := snapshot.SealingSegment() + if err != nil { + return fmt.Errorf("could not query sealing segment: %w", err) + } + + sealingSegmentLength := uint64(len(sealingSegment.AllBlocks())) + transactionExpiry := uint64(flow.DefaultTransactionExpiry) + blocksInSpork := head.Height - sporkRootBlockHeight + 1 // range is inclusive on both ends + + // Condition 3: + // check if head.Height - sporkRootBlockHeight < flow.DefaultTransactionExpiry + // this is the case where we bootstrap early into the spork and there is simply not enough blocks + if blocksInSpork < transactionExpiry { + // the distance to spork root is less than transaction expiry, we need all blocks back to the spork root. + if sealingSegmentLength != blocksInSpork { + return fmt.Errorf("invalid root snapshot length, expecting exactly (%d), got (%d)", blocksInSpork, sealingSegmentLength) + } + } else { + // Condition 2: + // the distance to spork root is more than transaction expiry, we need at least `transactionExpiry` many blocks + if sealingSegmentLength < transactionExpiry { + return fmt.Errorf("invalid root snapshot length, expecting at least (%d), got (%d)", + transactionExpiry, sealingSegmentLength) + } + } + return nil +} diff --git a/state/protocol/pebble/validity_test.go b/state/protocol/pebble/validity_test.go new file mode 100644 index 00000000000..53a044770c2 --- /dev/null +++ b/state/protocol/pebble/validity_test.go @@ -0,0 +1,234 @@ +package badger + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +var participants = unittest.IdentityListFixture(20, unittest.WithAllRoles()) + +func TestEpochSetupValidity(t *testing.T) { + t.Run("invalid first/final view", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // set an invalid final view for the first epoch + setup.FinalView = setup.FirstView + + err := verifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("non-canonically ordered identities", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // randomly shuffle the identities so they are not canonically ordered + var err error + setup.Participants, err = setup.Participants.Shuffle() + require.NoError(t, err) + err = verifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("invalid cluster assignments", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // create an invalid cluster assignment (node appears in multiple clusters) + collector := participants.Filter(filter.HasRole(flow.RoleCollection))[0] + setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) + + err := verifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("short seed", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + setup.RandomSource = unittest.SeedFixture(crypto.KeyGenSeedMinLen - 1) + + err := verifyEpochSetup(setup, true) + require.Error(t, err) + }) +} + +func TestBootstrapInvalidEpochCommit(t *testing.T) { + t.Run("inconsistent counter", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // use a different counter for the commit + commit.Counter = setup.Counter + 1 + + err := isValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("inconsistent cluster QCs", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // add an extra QC to commit + extraQC := unittest.QuorumCertificateWithSignerIDsFixture() + commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(extraQC)) + + err := isValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("missing dkg group key", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + commit.DKGGroupKey = nil + + err := isValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("inconsistent DKG participants", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // add an extra DKG participant key + commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) + + err := isValidEpochCommit(commit, setup) + require.Error(t, err) + }) +} + +// TestEntityExpirySnapshotValidation tests that we perform correct sanity checks when +// bootstrapping consensus nodes and access nodes we expect that we only bootstrap snapshots +// with sufficient history. +func TestEntityExpirySnapshotValidation(t *testing.T) { + t.Run("spork-root-snapshot", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("not-enough-history", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + rootSnapshot.Encodable().Head.Height += 10 // advance height to be not spork root snapshot + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.Error(t, err) + }) + t.Run("enough-history-spork-just-started", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot, but still lower than transaction expiry + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry / 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(int(flow.DefaultTransactionExpiry / 2)) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("enough-history-long-spork", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry * 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(int(flow.DefaultTransactionExpiry) - 1) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("more-history-than-needed", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry * 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(flow.DefaultTransactionExpiry * 2) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) +} + +func TestValidateVersionBeacon(t *testing.T) { + t.Run("no version beacon is ok", func(t *testing.T) { + snap := new(mock.Snapshot) + + snap.On("VersionBeacon").Return(nil, nil) + + err := validateVersionBeacon(snap) + require.NoError(t, err) + }) + t.Run("valid version beacon is ok", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 100 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 1000, + Version: "1.0.0", + }, + }, + Sequence: 50, + }, + SealHeight: uint64(37), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.NoError(t, err) + }) + t.Run("height must be below highest block", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 12 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 1000, + Version: "1.0.0", + }, + }, + Sequence: 50, + }, + SealHeight: uint64(37), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.Error(t, err) + require.True(t, protocol.IsInvalidServiceEventError(err)) + }) + t.Run("version beacon must be valid", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 12 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 0, + Version: "asdf", // invalid semver - hence will be considered invalid + }, + }, + Sequence: 50, + }, + SealHeight: uint64(1), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.Error(t, err) + require.True(t, protocol.IsInvalidServiceEventError(err)) + }) +} diff --git a/storage/pebble/all.go b/storage/pebble/all.go new file mode 100644 index 00000000000..58bc45e6848 --- /dev/null +++ b/storage/pebble/all.go @@ -0,0 +1,53 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/storage" +) + +func InitAll(metrics module.CacheMetrics, db *badger.DB) *storage.All { + headers := NewHeaders(metrics, db) + guarantees := NewGuarantees(metrics, db, DefaultCacheSize) + seals := NewSeals(metrics, db) + index := NewIndex(metrics, db) + results := NewExecutionResults(metrics, db) + receipts := NewExecutionReceipts(metrics, db, results, DefaultCacheSize) + payloads := NewPayloads(db, index, guarantees, seals, receipts, results) + blocks := NewBlocks(db, headers, payloads) + qcs := NewQuorumCertificates(metrics, db, DefaultCacheSize) + setups := NewEpochSetups(metrics, db) + epochCommits := NewEpochCommits(metrics, db) + statuses := NewEpochStatuses(metrics, db) + versionBeacons := NewVersionBeacons(db) + + commits := NewCommits(metrics, db) + transactions := NewTransactions(metrics, db) + transactionResults := NewTransactionResults(metrics, db, 10000) + collections := NewCollections(db, transactions) + events := NewEvents(metrics, db) + chunkDataPacks := NewChunkDataPacks(metrics, db, collections, 1000) + + return &storage.All{ + Headers: headers, + Guarantees: guarantees, + Seals: seals, + Index: index, + Payloads: payloads, + Blocks: blocks, + QuorumCertificates: qcs, + Setups: setups, + EpochCommits: epochCommits, + Statuses: statuses, + VersionBeacons: versionBeacons, + Results: results, + Receipts: receipts, + ChunkDataPacks: chunkDataPacks, + Commits: commits, + Transactions: transactions, + TransactionResults: transactionResults, + Collections: collections, + Events: events, + } +} diff --git a/storage/pebble/approvals.go b/storage/pebble/approvals.go new file mode 100644 index 00000000000..eb3cf4ae820 --- /dev/null +++ b/storage/pebble/approvals.go @@ -0,0 +1,136 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// ResultApprovals implements persistent storage for result approvals. +type ResultApprovals struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.ResultApproval] +} + +func NewResultApprovals(collector module.CacheMetrics, db *badger.DB) *ResultApprovals { + + store := func(key flow.Identifier, val *flow.ResultApproval) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertResultApproval(val))) + } + + retrieve := func(approvalID flow.Identifier) func(tx *badger.Txn) (*flow.ResultApproval, error) { + var approval flow.ResultApproval + return func(tx *badger.Txn) (*flow.ResultApproval, error) { + err := operation.RetrieveResultApproval(approvalID, &approval)(tx) + return &approval, err + } + } + + res := &ResultApprovals{ + db: db, + cache: newCache[flow.Identifier, *flow.ResultApproval](collector, metrics.ResourceResultApprovals, + withLimit[flow.Identifier, *flow.ResultApproval](flow.DefaultTransactionExpiry+100), + withStore[flow.Identifier, *flow.ResultApproval](store), + withRetrieve[flow.Identifier, *flow.ResultApproval](retrieve)), + } + + return res +} + +func (r *ResultApprovals) store(approval *flow.ResultApproval) func(*transaction.Tx) error { + return r.cache.PutTx(approval.ID(), approval) +} + +func (r *ResultApprovals) byID(approvalID flow.Identifier) func(*badger.Txn) (*flow.ResultApproval, error) { + return func(tx *badger.Txn) (*flow.ResultApproval, error) { + val, err := r.cache.Get(approvalID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +func (r *ResultApprovals) byChunk(resultID flow.Identifier, chunkIndex uint64) func(*badger.Txn) (*flow.ResultApproval, error) { + return func(tx *badger.Txn) (*flow.ResultApproval, error) { + var approvalID flow.Identifier + err := operation.LookupResultApproval(resultID, chunkIndex, &approvalID)(tx) + if err != nil { + return nil, fmt.Errorf("could not lookup result approval ID: %w", err) + } + return r.byID(approvalID)(tx) + } +} + +func (r *ResultApprovals) index(resultID flow.Identifier, chunkIndex uint64, approvalID flow.Identifier) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + err := operation.IndexResultApproval(resultID, chunkIndex, approvalID)(tx) + if err == nil { + return nil + } + + if !errors.Is(err, storage.ErrAlreadyExists) { + return err + } + + // When trying to index an approval for a result, and there is already + // an approval for the result, double check if the indexed approval is + // the same. + // We don't allow indexing multiple approvals per chunk because the + // store is only used within Verification nodes, and it is impossible + // for a Verification node to compute different approvals for the same + // chunk. + var storedApprovalID flow.Identifier + err = operation.LookupResultApproval(resultID, chunkIndex, &storedApprovalID)(tx) + if err != nil { + return fmt.Errorf("there is an approval stored already, but cannot retrieve it: %w", err) + } + + if storedApprovalID != approvalID { + return fmt.Errorf("attempting to store conflicting approval (result: %v, chunk index: %d): storing: %v, stored: %v. %w", + resultID, chunkIndex, approvalID, storedApprovalID, storage.ErrDataMismatch) + } + + return nil + } +} + +// Store stores a ResultApproval +func (r *ResultApprovals) Store(approval *flow.ResultApproval) error { + return operation.RetryOnConflictTx(r.db, transaction.Update, r.store(approval)) +} + +// Index indexes a ResultApproval by chunk (ResultID + chunk index). +// operation is idempotent (repeated calls with the same value are equivalent to +// just calling the method once; still the method succeeds on each call). +func (r *ResultApprovals) Index(resultID flow.Identifier, chunkIndex uint64, approvalID flow.Identifier) error { + err := operation.RetryOnConflict(r.db.Update, r.index(resultID, chunkIndex, approvalID)) + if err != nil { + return fmt.Errorf("could not index result approval: %w", err) + } + return nil +} + +// ByID retrieves a ResultApproval by its ID +func (r *ResultApprovals) ByID(approvalID flow.Identifier) (*flow.ResultApproval, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byID(approvalID)(tx) +} + +// ByChunk retrieves a ResultApproval by result ID and chunk index. The +// ResultApprovals store is only used within a verification node, where it is +// assumed that there is never more than one approval per chunk. +func (r *ResultApprovals) ByChunk(resultID flow.Identifier, chunkIndex uint64) (*flow.ResultApproval, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byChunk(resultID, chunkIndex)(tx) +} diff --git a/storage/pebble/approvals_test.go b/storage/pebble/approvals_test.go new file mode 100644 index 00000000000..1b13a49ae59 --- /dev/null +++ b/storage/pebble/approvals_test.go @@ -0,0 +1,81 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestApprovalStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewResultApprovals(metrics, db) + + approval := unittest.ResultApprovalFixture() + err := store.Store(approval) + require.NoError(t, err) + + err = store.Index(approval.Body.ExecutionResultID, approval.Body.ChunkIndex, approval.ID()) + require.NoError(t, err) + + byID, err := store.ByID(approval.ID()) + require.NoError(t, err) + require.Equal(t, approval, byID) + + byChunk, err := store.ByChunk(approval.Body.ExecutionResultID, approval.Body.ChunkIndex) + require.NoError(t, err) + require.Equal(t, approval, byChunk) + }) +} + +func TestApprovalStoreTwice(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewResultApprovals(metrics, db) + + approval := unittest.ResultApprovalFixture() + err := store.Store(approval) + require.NoError(t, err) + + err = store.Index(approval.Body.ExecutionResultID, approval.Body.ChunkIndex, approval.ID()) + require.NoError(t, err) + + err = store.Store(approval) + require.NoError(t, err) + + err = store.Index(approval.Body.ExecutionResultID, approval.Body.ChunkIndex, approval.ID()) + require.NoError(t, err) + }) +} + +func TestApprovalStoreTwoDifferentApprovalsShouldFail(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewResultApprovals(metrics, db) + + approval1 := unittest.ResultApprovalFixture() + approval2 := unittest.ResultApprovalFixture() + + err := store.Store(approval1) + require.NoError(t, err) + + err = store.Index(approval1.Body.ExecutionResultID, approval1.Body.ChunkIndex, approval1.ID()) + require.NoError(t, err) + + // we can store a different approval, but we can't index a different + // approval for the same chunk. + err = store.Store(approval2) + require.NoError(t, err) + + err = store.Index(approval1.Body.ExecutionResultID, approval1.Body.ChunkIndex, approval2.ID()) + require.Error(t, err) + require.True(t, errors.Is(err, storage.ErrDataMismatch)) + }) +} diff --git a/storage/pebble/blocks.go b/storage/pebble/blocks.go new file mode 100644 index 00000000000..9d3b64a1ffc --- /dev/null +++ b/storage/pebble/blocks.go @@ -0,0 +1,155 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// Blocks implements a simple block storage around a badger DB. +type Blocks struct { + db *badger.DB + headers *Headers + payloads *Payloads +} + +// NewBlocks ... +func NewBlocks(db *badger.DB, headers *Headers, payloads *Payloads) *Blocks { + b := &Blocks{ + db: db, + headers: headers, + payloads: payloads, + } + return b +} + +func (b *Blocks) StoreTx(block *flow.Block) func(*transaction.Tx) error { + return func(tx *transaction.Tx) error { + err := b.headers.storeTx(block.Header)(tx) + if err != nil { + return fmt.Errorf("could not store header %v: %w", block.Header.ID(), err) + } + err = b.payloads.storeTx(block.ID(), block.Payload)(tx) + if err != nil { + return fmt.Errorf("could not store payload: %w", err) + } + return nil + } +} + +func (b *Blocks) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.Block, error) { + return func(tx *badger.Txn) (*flow.Block, error) { + header, err := b.headers.retrieveTx(blockID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve header: %w", err) + } + payload, err := b.payloads.retrieveTx(blockID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve payload: %w", err) + } + block := &flow.Block{ + Header: header, + Payload: payload, + } + return block, nil + } +} + +// Store ... +func (b *Blocks) Store(block *flow.Block) error { + return operation.RetryOnConflictTx(b.db, transaction.Update, b.StoreTx(block)) +} + +// ByID ... +func (b *Blocks) ByID(blockID flow.Identifier) (*flow.Block, error) { + tx := b.db.NewTransaction(false) + defer tx.Discard() + return b.retrieveTx(blockID)(tx) +} + +// ByHeight ... +func (b *Blocks) ByHeight(height uint64) (*flow.Block, error) { + tx := b.db.NewTransaction(false) + defer tx.Discard() + + blockID, err := b.headers.retrieveIdByHeightTx(height)(tx) + if err != nil { + return nil, err + } + return b.retrieveTx(blockID)(tx) +} + +// ByCollectionID ... +func (b *Blocks) ByCollectionID(collID flow.Identifier) (*flow.Block, error) { + var blockID flow.Identifier + err := b.db.View(operation.LookupCollectionBlock(collID, &blockID)) + if err != nil { + return nil, fmt.Errorf("could not look up block: %w", err) + } + return b.ByID(blockID) +} + +// IndexBlockForCollections ... +func (b *Blocks) IndexBlockForCollections(blockID flow.Identifier, collIDs []flow.Identifier) error { + for _, collID := range collIDs { + err := operation.RetryOnConflict(b.db.Update, operation.SkipDuplicates(operation.IndexCollectionBlock(collID, blockID))) + if err != nil { + return fmt.Errorf("could not index collection block (%x): %w", collID, err) + } + } + return nil +} + +// InsertLastFullBlockHeightIfNotExists inserts the last full block height +// Calling this function multiple times is a no-op and returns no expected errors. +func (b *Blocks) InsertLastFullBlockHeightIfNotExists(height uint64) error { + return operation.RetryOnConflict(b.db.Update, func(tx *badger.Txn) error { + err := operation.InsertLastCompleteBlockHeightIfNotExists(height)(tx) + if err != nil { + return fmt.Errorf("could not set LastFullBlockHeight: %w", err) + } + return nil + }) +} + +// UpdateLastFullBlockHeight upsert (update or insert) the last full block height +func (b *Blocks) UpdateLastFullBlockHeight(height uint64) error { + return operation.RetryOnConflict(b.db.Update, func(tx *badger.Txn) error { + + // try to update + err := operation.UpdateLastCompleteBlockHeight(height)(tx) + if err == nil { + return nil + } + + if !errors.Is(err, storage.ErrNotFound) { + return fmt.Errorf("could not update LastFullBlockHeight: %w", err) + } + + // if key does not exist, try insert. + err = operation.InsertLastCompleteBlockHeight(height)(tx) + if err != nil { + return fmt.Errorf("could not insert LastFullBlockHeight: %w", err) + } + + return nil + }) +} + +// GetLastFullBlockHeight ... +func (b *Blocks) GetLastFullBlockHeight() (uint64, error) { + var h uint64 + err := b.db.View(operation.RetrieveLastCompleteBlockHeight(&h)) + if err != nil { + return 0, fmt.Errorf("failed to retrieve LastFullBlockHeight: %w", err) + } + return h, nil +} diff --git a/storage/pebble/blocks_test.go b/storage/pebble/blocks_test.go new file mode 100644 index 00000000000..d459f00751d --- /dev/null +++ b/storage/pebble/blocks_test.go @@ -0,0 +1,72 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestBlocks(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + store := badgerstorage.NewBlocks(db, nil, nil) + + // check retrieval of non-existing key + _, err := store.GetLastFullBlockHeight() + assert.Error(t, err) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // insert a value for height + var height1 = uint64(1234) + err = store.UpdateLastFullBlockHeight(height1) + assert.NoError(t, err) + + // check value can be retrieved + actual, err := store.GetLastFullBlockHeight() + assert.NoError(t, err) + assert.Equal(t, height1, actual) + + // update the value for height + var height2 = uint64(1234) + err = store.UpdateLastFullBlockHeight(height2) + assert.NoError(t, err) + + // check that the new value can be retrieved + actual, err = store.GetLastFullBlockHeight() + assert.NoError(t, err) + assert.Equal(t, height2, actual) + }) +} + +func TestBlockStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + cacheMetrics := &metrics.NoopCollector{} + // verify after storing a block should be able to retrieve it back + blocks := badgerstorage.InitAll(cacheMetrics, db).Blocks + block := unittest.FullBlockFixture() + block.SetPayload(unittest.PayloadFixture(unittest.WithAllTheFixins)) + + err := blocks.Store(&block) + require.NoError(t, err) + + retrieved, err := blocks.ByID(block.ID()) + require.NoError(t, err) + + require.Equal(t, &block, retrieved) + + // verify after a restart, the block stored in the database is the same + // as the original + blocksAfterRestart := badgerstorage.InitAll(cacheMetrics, db).Blocks + receivedAfterRestart, err := blocksAfterRestart.ByID(block.ID()) + require.NoError(t, err) + + require.Equal(t, &block, receivedAfterRestart) + }) +} diff --git a/storage/pebble/cache_test.go b/storage/pebble/cache_test.go new file mode 100644 index 00000000000..76ea7ce18bc --- /dev/null +++ b/storage/pebble/cache_test.go @@ -0,0 +1,40 @@ +package badger + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestCache_Exists tests existence checking items in the cache. +func TestCache_Exists(t *testing.T) { + cache := newCache[flow.Identifier, any](metrics.NewNoopCollector(), "test") + + t.Run("non-existent", func(t *testing.T) { + key := unittest.IdentifierFixture() + exists := cache.IsCached(key) + assert.False(t, exists) + }) + + t.Run("existent", func(t *testing.T) { + key := unittest.IdentifierFixture() + cache.Insert(key, unittest.RandomBytes(128)) + + exists := cache.IsCached(key) + assert.True(t, exists) + }) + + t.Run("removed", func(t *testing.T) { + key := unittest.IdentifierFixture() + // insert, then remove the item + cache.Insert(key, unittest.RandomBytes(128)) + cache.Remove(key) + + exists := cache.IsCached(key) + assert.False(t, exists) + }) +} diff --git a/storage/pebble/chunkDataPacks.go b/storage/pebble/chunkDataPacks.go new file mode 100644 index 00000000000..05f42cf7856 --- /dev/null +++ b/storage/pebble/chunkDataPacks.go @@ -0,0 +1,155 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type ChunkDataPacks struct { + db *badger.DB + collections storage.Collections + byChunkIDCache *Cache[flow.Identifier, *storage.StoredChunkDataPack] +} + +func NewChunkDataPacks(collector module.CacheMetrics, db *badger.DB, collections storage.Collections, byChunkIDCacheSize uint) *ChunkDataPacks { + + store := func(key flow.Identifier, val *storage.StoredChunkDataPack) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(val))) + } + + retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + var c storage.StoredChunkDataPack + err := operation.RetrieveChunkDataPack(key, &c)(tx) + return &c, err + } + } + + cache := newCache(collector, metrics.ResourceChunkDataPack, + withLimit[flow.Identifier, *storage.StoredChunkDataPack](byChunkIDCacheSize), + withStore(store), + withRetrieve(retrieve), + ) + + ch := ChunkDataPacks{ + db: db, + byChunkIDCache: cache, + collections: collections, + } + return &ch +} + +// Remove removes multiple ChunkDataPacks cs keyed by their ChunkIDs in a batch. +// No errors are expected during normal operation, even if no entries are matched. +func (ch *ChunkDataPacks) Remove(chunkIDs []flow.Identifier) error { + batch := NewBatch(ch.db) + + for _, c := range chunkIDs { + err := ch.BatchRemove(c, batch) + if err != nil { + return fmt.Errorf("cannot remove chunk data pack: %w", err) + } + } + + err := batch.Flush() + if err != nil { + return fmt.Errorf("cannot flush batch to remove chunk data pack: %w", err) + } + return nil +} + +// BatchStore stores ChunkDataPack c keyed by its ChunkID in provided batch. +// No errors are expected during normal operation, but it may return generic error +// if entity is not serializable or Badger unexpectedly fails to process request +func (ch *ChunkDataPacks) BatchStore(c *flow.ChunkDataPack, batch storage.BatchStorage) error { + sc := storage.ToStoredChunkDataPack(c) + writeBatch := batch.GetWriter() + batch.OnSucceed(func() { + ch.byChunkIDCache.Insert(sc.ChunkID, sc) + }) + return operation.BatchInsertChunkDataPack(sc)(writeBatch) +} + +// Store stores multiple ChunkDataPacks cs keyed by their ChunkIDs in a batch. +// No errors are expected during normal operation, but it may return generic error +func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { + batch := NewBatch(ch.db) + for _, c := range cs { + err := ch.BatchStore(c, batch) + if err != nil { + return fmt.Errorf("cannot store chunk data pack: %w", err) + } + } + + err := batch.Flush() + if err != nil { + return fmt.Errorf("cannot flush batch: %w", err) + } + return nil +} + +// BatchRemove removes ChunkDataPack c keyed by its ChunkID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + batch.OnSucceed(func() { + ch.byChunkIDCache.Remove(chunkID) + }) + return operation.BatchRemoveChunkDataPack(chunkID)(writeBatch) +} + +func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { + schdp, err := ch.byChunkID(chunkID) + if err != nil { + return nil, err + } + + chdp := &flow.ChunkDataPack{ + ChunkID: schdp.ChunkID, + StartState: schdp.StartState, + Proof: schdp.Proof, + ExecutionDataRoot: schdp.ExecutionDataRoot, + } + + if !schdp.SystemChunk { + collection, err := ch.collections.ByID(schdp.CollectionID) + if err != nil { + return nil, fmt.Errorf("could not retrive collection (id: %x) for stored chunk data pack: %w", schdp.CollectionID, err) + } + + chdp.Collection = collection + } + + return chdp, nil +} + +func (ch *ChunkDataPacks) byChunkID(chunkID flow.Identifier) (*storage.StoredChunkDataPack, error) { + tx := ch.db.NewTransaction(false) + defer tx.Discard() + + schdp, err := ch.retrieveCHDP(chunkID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrive stored chunk data pack: %w", err) + } + + return schdp, nil +} + +func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn) (*storage.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + val, err := ch.byChunkIDCache.Get(chunkID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} diff --git a/storage/pebble/chunk_consumer_test.go b/storage/pebble/chunk_consumer_test.go new file mode 100644 index 00000000000..05af3a1ca29 --- /dev/null +++ b/storage/pebble/chunk_consumer_test.go @@ -0,0 +1,11 @@ +package badger + +import "testing" + +// 1. can init +// 2. can't set a process if never inited +// 3. can set after init +// 4. can read after init +// 5. can read after set +func TestChunkConsumer(t *testing.T) { +} diff --git a/storage/pebble/chunk_data_pack_test.go b/storage/pebble/chunk_data_pack_test.go new file mode 100644 index 00000000000..0a98e9d170d --- /dev/null +++ b/storage/pebble/chunk_data_pack_test.go @@ -0,0 +1,143 @@ +package badger_test + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestChunkDataPacks_Store evaluates correct storage and retrieval of chunk data packs in the storage. +// It also evaluates that re-inserting is idempotent. +func TestChunkDataPacks_Store(t *testing.T) { + WithChunkDataPacks(t, 100, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore *badgerstorage.ChunkDataPacks, _ *badger.DB) { + require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + }) +} + +func TestChunkDataPack_Remove(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, db) + collections := badgerstorage.NewCollections(db, transactions) + // keep the cache size at 1 to make sure that entries are written and read from storage itself. + chunkDataPackStore := badgerstorage.NewChunkDataPacks(&metrics.NoopCollector{}, db, collections, 1) + + chunkDataPacks := unittest.ChunkDataPacksFixture(10) + for _, chunkDataPack := range chunkDataPacks { + // stores collection in Collections storage (which ChunkDataPacks store uses internally) + err := collections.Store(chunkDataPack.Collection) + require.NoError(t, err) + } + + chunkIDs := make([]flow.Identifier, 0, len(chunkDataPacks)) + for _, chunk := range chunkDataPacks { + chunkIDs = append(chunkIDs, chunk.ID()) + } + + require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + require.NoError(t, chunkDataPackStore.Remove(chunkIDs)) + + // verify it has been removed + _, err := chunkDataPackStore.ByChunkID(chunkIDs[0]) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // Removing again should not error + require.NoError(t, chunkDataPackStore.Remove(chunkIDs)) + }) +} + +// TestChunkDataPack_BatchStore evaluates correct batch storage and retrieval of chunk data packs in the storage. +func TestChunkDataPacks_BatchStore(t *testing.T) { + WithChunkDataPacks(t, 100, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore *badgerstorage.ChunkDataPacks, db *badger.DB) { + batch := badgerstorage.NewBatch(db) + + wg := sync.WaitGroup{} + wg.Add(len(chunkDataPacks)) + for _, chunkDataPack := range chunkDataPacks { + go func(cdp flow.ChunkDataPack) { + err := chunkDataPackStore.BatchStore(&cdp, batch) + require.NoError(t, err) + + wg.Done() + }(*chunkDataPack) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 1*time.Second, "could not store chunk data packs on time") + + err := batch.Flush() + require.NoError(t, err) + }) +} + +// TestChunkDataPacks_MissingItem evaluates querying a missing item returns a storage.ErrNotFound error. +func TestChunkDataPacks_MissingItem(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, db) + collections := badgerstorage.NewCollections(db, transactions) + store := badgerstorage.NewChunkDataPacks(&metrics.NoopCollector{}, db, collections, 1) + + // attempt to get an invalid + _, err := store.ByChunkID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} + +// TestChunkDataPacks_StoreTwice evaluates that storing the same chunk data pack twice +// does not result in an error. +func TestChunkDataPacks_StoreTwice(t *testing.T) { + WithChunkDataPacks(t, 2, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore *badgerstorage.ChunkDataPacks, db *badger.DB) { + transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, db) + collections := badgerstorage.NewCollections(db, transactions) + store := badgerstorage.NewChunkDataPacks(&metrics.NoopCollector{}, db, collections, 1) + require.NoError(t, store.Store(chunkDataPacks)) + + for _, c := range chunkDataPacks { + c2, err := store.ByChunkID(c.ChunkID) + require.NoError(t, err) + require.Equal(t, c, c2) + } + + require.NoError(t, store.Store(chunkDataPacks)) + }) +} + +// WithChunkDataPacks is a test helper that generates specified number of chunk data packs, store them using the storeFunc, and +// then evaluates whether they are successfully retrieved from storage. +func WithChunkDataPacks(t *testing.T, chunks int, storeFunc func(*testing.T, []*flow.ChunkDataPack, *badgerstorage.ChunkDataPacks, *badger.DB)) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, db) + collections := badgerstorage.NewCollections(db, transactions) + // keep the cache size at 1 to make sure that entries are written and read from storage itself. + store := badgerstorage.NewChunkDataPacks(&metrics.NoopCollector{}, db, collections, 1) + + chunkDataPacks := unittest.ChunkDataPacksFixture(chunks) + for _, chunkDataPack := range chunkDataPacks { + // stores collection in Collections storage (which ChunkDataPacks store uses internally) + err := collections.Store(chunkDataPack.Collection) + require.NoError(t, err) + } + + // stores chunk data packs in the memory using provided store function. + storeFunc(t, chunkDataPacks, store, db) + + // stored chunk data packs should be retrieved successfully. + for _, expected := range chunkDataPacks { + actual, err := store.ByChunkID(expected.ChunkID) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + } + }) +} diff --git a/storage/pebble/chunks_queue.go b/storage/pebble/chunks_queue.go new file mode 100644 index 00000000000..430abe0241b --- /dev/null +++ b/storage/pebble/chunks_queue.go @@ -0,0 +1,117 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/chunks" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// ChunksQueue stores a queue of chunk locators that assigned to me to verify. +// Job consumers can read the locators as job from the queue by index. +// Chunk locators stored in this queue are unique. +type ChunksQueue struct { + db *badger.DB +} + +const JobQueueChunksQueue = "JobQueueChunksQueue" + +// NewChunkQueue will initialize the underlying badger database of chunk locator queue. +func NewChunkQueue(db *badger.DB) *ChunksQueue { + return &ChunksQueue{ + db: db, + } +} + +// Init initializes chunk queue's latest index with the given default index. +func (q *ChunksQueue) Init(defaultIndex uint64) (bool, error) { + _, err := q.LatestIndex() + if errors.Is(err, storage.ErrNotFound) { + err = q.db.Update(operation.InitJobLatestIndex(JobQueueChunksQueue, defaultIndex)) + if err != nil { + return false, fmt.Errorf("could not init chunk locator queue with default index %v: %w", defaultIndex, err) + } + return true, nil + } + if err != nil { + return false, fmt.Errorf("could not get latest index: %w", err) + } + + return false, nil +} + +// StoreChunkLocator stores a new chunk locator that assigned to me to the job queue. +// A true will be returned, if the locator was new. +// A false will be returned, if the locator was duplicate. +func (q *ChunksQueue) StoreChunkLocator(locator *chunks.Locator) (bool, error) { + err := operation.RetryOnConflict(q.db.Update, func(tx *badger.Txn) error { + // make sure the chunk locator is unique + err := operation.InsertChunkLocator(locator)(tx) + if err != nil { + return fmt.Errorf("failed to insert chunk locator: %w", err) + } + + // read the latest index + var latest uint64 + err = operation.RetrieveJobLatestIndex(JobQueueChunksQueue, &latest)(tx) + if err != nil { + return fmt.Errorf("failed to retrieve job index for chunk locator queue: %w", err) + } + + // insert to the next index + next := latest + 1 + err = operation.InsertJobAtIndex(JobQueueChunksQueue, next, locator.ID())(tx) + if err != nil { + return fmt.Errorf("failed to set job index for chunk locator queue at index %v: %w", next, err) + } + + // update the next index as the latest index + err = operation.SetJobLatestIndex(JobQueueChunksQueue, next)(tx) + if err != nil { + return fmt.Errorf("failed to update latest index %v: %w", next, err) + } + + return nil + }) + + // was trying to store a duplicate locator + if errors.Is(err, storage.ErrAlreadyExists) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("failed to store chunk locator: %w", err) + } + return true, nil +} + +// LatestIndex returns the index of the latest chunk locator stored in the queue. +func (q *ChunksQueue) LatestIndex() (uint64, error) { + var latest uint64 + err := q.db.View(operation.RetrieveJobLatestIndex(JobQueueChunksQueue, &latest)) + if err != nil { + return 0, fmt.Errorf("could not retrieve latest index for chunks queue: %w", err) + } + return latest, nil +} + +// AtIndex returns the chunk locator stored at the given index in the queue. +func (q *ChunksQueue) AtIndex(index uint64) (*chunks.Locator, error) { + var locatorID flow.Identifier + err := q.db.View(operation.RetrieveJobAtIndex(JobQueueChunksQueue, index, &locatorID)) + if err != nil { + return nil, fmt.Errorf("could not retrieve chunk locator in queue: %w", err) + } + + var locator chunks.Locator + err = q.db.View(operation.RetrieveChunkLocator(locatorID, &locator)) + if err != nil { + return nil, fmt.Errorf("could not retrieve locator for chunk id %v: %w", locatorID, err) + } + + return &locator, nil +} diff --git a/storage/pebble/chunks_queue_test.go b/storage/pebble/chunks_queue_test.go new file mode 100644 index 00000000000..e1e9350afe8 --- /dev/null +++ b/storage/pebble/chunks_queue_test.go @@ -0,0 +1,16 @@ +package badger + +import "testing" + +// 1. should be able to read after store +// 2. should be able to read the latest index after store +// 3. should return false if a duplicate chunk is stored +// 4. should return true if a new chunk is stored +// 5. should return an increased index when a chunk is stored +// 6. storing 100 chunks concurrent should return last index as 100 +// 7. should not be able to read with wrong index +// 8. should return init index after init +// 9. storing chunk and updating the latest index should be atomic +func TestStoreAndRead(t *testing.T) { + // TODO +} diff --git a/storage/pebble/cleaner.go b/storage/pebble/cleaner.go new file mode 100644 index 00000000000..d9cd07997e7 --- /dev/null +++ b/storage/pebble/cleaner.go @@ -0,0 +1,122 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "time" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/utils/rand" +) + +// Cleaner uses component.ComponentManager to implement module.Startable and module.ReadyDoneAware +// to run an internal goroutine which run badger value log garbage collection at a semi-regular interval. +// The Cleaner exists for 2 reasons: +// - Run GC frequently enough that each GC is relatively inexpensive +// - Avoid GC being synchronized across all nodes. Since in the happy path, all nodes have very similar +// database load patterns, without intervention they are likely to schedule GC at the same time, which +// can cause temporary consensus halts. +type Cleaner struct { + component.Component + log zerolog.Logger + db *badger.DB + metrics module.CleanerMetrics + ratio float64 + interval time.Duration +} + +var _ component.Component = (*Cleaner)(nil) + +// NewCleaner returns a cleaner that runs the badger value log garbage collection once every `interval` duration +// if an interval of zero is passed in, we will not run the GC at all. +func NewCleaner(log zerolog.Logger, db *badger.DB, metrics module.CleanerMetrics, interval time.Duration) *Cleaner { + // NOTE: we run garbage collection frequently at points in our business + // logic where we are likely to have a small breather in activity; it thus + // makes sense to run garbage collection often, with a smaller ratio, rather + // than running it rarely and having big rewrites at once + c := &Cleaner{ + log: log.With().Str("component", "cleaner").Logger(), + db: db, + metrics: metrics, + ratio: 0.2, + interval: interval, + } + + // Disable if passed in 0 as interval + if c.interval == 0 { + c.Component = &module.NoopComponent{} + return c + } + + c.Component = component.NewComponentManagerBuilder(). + AddWorker(c.gcWorkerRoutine). + Build() + + return c +} + +// gcWorkerRoutine runs badger GC on timely basis. +func (c *Cleaner) gcWorkerRoutine(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + ticker := time.NewTicker(c.nextWaitDuration()) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.runGC() + + // reset the ticker with a new interval and random jitter + ticker.Reset(c.nextWaitDuration()) + } + } +} + +// nextWaitDuration calculates next duration for Cleaner to wait before attempting to run GC. +// We add 20% jitter into the interval, so that we don't risk nodes syncing their GC calls over time. +// Therefore GC is run every X seconds, where X is uniformly sampled from [interval, interval*1.2] +func (c *Cleaner) nextWaitDuration() time.Duration { + jitter, err := rand.Uint64n(uint64(c.interval.Nanoseconds() / 5)) + if err != nil { + // if randomness fails, do not use a jitter for this instance. + // TODO: address the error properly and not swallow it. + // In this specific case, `utils/rand` only errors if the system randomness fails + // which is a symptom of a wider failure. Many other node components would catch such + // a failure. + c.log.Warn().Msg("jitter is zero beacuse system randomness has failed") + jitter = 0 + } + return time.Duration(c.interval.Nanoseconds() + int64(jitter)) +} + +// runGC runs garbage collection for badger DB, handles sentinel errors and reports metrics. +func (c *Cleaner) runGC() { + started := time.Now() + err := c.db.RunValueLogGC(c.ratio) + if err == badger.ErrRejected { + // NOTE: this happens when a GC call is already running + c.log.Warn().Msg("garbage collection on value log already running") + return + } + if err == badger.ErrNoRewrite { + // NOTE: this happens when no files have any garbage to drop + c.log.Debug().Msg("garbage collection on value log unnecessary") + return + } + if err != nil { + c.log.Error().Err(err).Msg("garbage collection on value log failed") + return + } + + runtime := time.Since(started) + c.log.Debug(). + Dur("gc_duration", runtime). + Msg("garbage collection on value log executed") + c.metrics.RanGC(runtime) +} diff --git a/storage/pebble/cluster_blocks.go b/storage/pebble/cluster_blocks.go new file mode 100644 index 00000000000..88aef54526f --- /dev/null +++ b/storage/pebble/cluster_blocks.go @@ -0,0 +1,73 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// ClusterBlocks implements a simple block storage around a badger DB. +type ClusterBlocks struct { + db *badger.DB + chainID flow.ChainID + headers *Headers + payloads *ClusterPayloads +} + +func NewClusterBlocks(db *badger.DB, chainID flow.ChainID, headers *Headers, payloads *ClusterPayloads) *ClusterBlocks { + b := &ClusterBlocks{ + db: db, + chainID: chainID, + headers: headers, + payloads: payloads, + } + return b +} + +func (b *ClusterBlocks) Store(block *cluster.Block) error { + return operation.RetryOnConflictTx(b.db, transaction.Update, b.storeTx(block)) +} + +func (b *ClusterBlocks) storeTx(block *cluster.Block) func(*transaction.Tx) error { + return func(tx *transaction.Tx) error { + err := b.headers.storeTx(block.Header)(tx) + if err != nil { + return fmt.Errorf("could not store header: %w", err) + } + err = b.payloads.storeTx(block.ID(), block.Payload)(tx) + if err != nil { + return fmt.Errorf("could not store payload: %w", err) + } + return nil + } +} + +func (b *ClusterBlocks) ByID(blockID flow.Identifier) (*cluster.Block, error) { + header, err := b.headers.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("could not get header: %w", err) + } + payload, err := b.payloads.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("could not retrieve payload: %w", err) + } + block := cluster.Block{ + Header: header, + Payload: payload, + } + return &block, nil +} + +func (b *ClusterBlocks) ByHeight(height uint64) (*cluster.Block, error) { + var blockID flow.Identifier + err := b.db.View(operation.LookupClusterBlockHeight(b.chainID, height, &blockID)) + if err != nil { + return nil, fmt.Errorf("could not look up block: %w", err) + } + return b.ByID(blockID) +} diff --git a/storage/pebble/cluster_blocks_test.go b/storage/pebble/cluster_blocks_test.go new file mode 100644 index 00000000000..64def9fec6b --- /dev/null +++ b/storage/pebble/cluster_blocks_test.go @@ -0,0 +1,50 @@ +package badger + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestClusterBlocksByHeight(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + chain := unittest.ClusterBlockChainFixture(5) + parent, blocks := chain[0], chain[1:] + + // add parent as boundary + err := db.Update(operation.IndexClusterBlockHeight(parent.Header.ChainID, parent.Header.Height, parent.ID())) + require.NoError(t, err) + + err = db.Update(operation.InsertClusterFinalizedHeight(parent.Header.ChainID, parent.Header.Height)) + require.NoError(t, err) + + // store a chain of blocks + for _, block := range blocks { + err := db.Update(procedure.InsertClusterBlock(&block)) + require.NoError(t, err) + + err = db.Update(procedure.FinalizeClusterBlock(block.Header.ID())) + require.NoError(t, err) + } + + clusterBlocks := NewClusterBlocks( + db, + blocks[0].Header.ChainID, + NewHeaders(metrics.NewNoopCollector(), db), + NewClusterPayloads(metrics.NewNoopCollector(), db), + ) + + // check if the block can be retrieved by height + for _, block := range blocks { + retrievedBlock, err := clusterBlocks.ByHeight(block.Header.Height) + require.NoError(t, err) + require.Equal(t, block.ID(), retrievedBlock.ID()) + } + }) +} diff --git a/storage/pebble/cluster_payloads.go b/storage/pebble/cluster_payloads.go new file mode 100644 index 00000000000..0fc3ba3ee28 --- /dev/null +++ b/storage/pebble/cluster_payloads.go @@ -0,0 +1,68 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// ClusterPayloads implements storage of block payloads for collection node +// cluster consensus. +type ClusterPayloads struct { + db *badger.DB + cache *Cache[flow.Identifier, *cluster.Payload] +} + +func NewClusterPayloads(cacheMetrics module.CacheMetrics, db *badger.DB) *ClusterPayloads { + + store := func(blockID flow.Identifier, payload *cluster.Payload) func(*transaction.Tx) error { + return transaction.WithTx(procedure.InsertClusterPayload(blockID, payload)) + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*cluster.Payload, error) { + var payload cluster.Payload + return func(tx *badger.Txn) (*cluster.Payload, error) { + err := procedure.RetrieveClusterPayload(blockID, &payload)(tx) + return &payload, err + } + } + + cp := &ClusterPayloads{ + db: db, + cache: newCache[flow.Identifier, *cluster.Payload](cacheMetrics, metrics.ResourceClusterPayload, + withLimit[flow.Identifier, *cluster.Payload](flow.DefaultTransactionExpiry*4), + withStore(store), + withRetrieve(retrieve)), + } + + return cp +} + +func (cp *ClusterPayloads) storeTx(blockID flow.Identifier, payload *cluster.Payload) func(*transaction.Tx) error { + return cp.cache.PutTx(blockID, payload) +} +func (cp *ClusterPayloads) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*cluster.Payload, error) { + return func(tx *badger.Txn) (*cluster.Payload, error) { + val, err := cp.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +func (cp *ClusterPayloads) Store(blockID flow.Identifier, payload *cluster.Payload) error { + return operation.RetryOnConflictTx(cp.db, transaction.Update, cp.storeTx(blockID, payload)) +} + +func (cp *ClusterPayloads) ByBlockID(blockID flow.Identifier) (*cluster.Payload, error) { + tx := cp.db.NewTransaction(false) + defer tx.Discard() + return cp.retrieveTx(blockID)(tx) +} diff --git a/storage/pebble/cluster_payloads_test.go b/storage/pebble/cluster_payloads_test.go new file mode 100644 index 00000000000..797c0c701fa --- /dev/null +++ b/storage/pebble/cluster_payloads_test.go @@ -0,0 +1,51 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestStoreRetrieveClusterPayload(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewClusterPayloads(metrics, db) + + blockID := unittest.IdentifierFixture() + expected := unittest.ClusterPayloadFixture(5) + + // store payload + err := store.Store(blockID, expected) + require.NoError(t, err) + + // fetch payload + payload, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Equal(t, expected, payload) + + // storing again should error with key already exists + err = store.Store(blockID, expected) + require.True(t, errors.Is(err, storage.ErrAlreadyExists)) + }) +} + +func TestClusterPayloadRetrieveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewClusterPayloads(metrics, db) + + blockID := unittest.IdentifierFixture() + + _, err := store.ByBlockID(blockID) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} diff --git a/storage/pebble/collections.go b/storage/pebble/collections.go new file mode 100644 index 00000000000..748d4a04c74 --- /dev/null +++ b/storage/pebble/collections.go @@ -0,0 +1,156 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type Collections struct { + db *badger.DB + transactions *Transactions +} + +func NewCollections(db *badger.DB, transactions *Transactions) *Collections { + c := &Collections{ + db: db, + transactions: transactions, + } + return c +} + +func (c *Collections) StoreLight(collection *flow.LightCollection) error { + err := operation.RetryOnConflict(c.db.Update, operation.InsertCollection(collection)) + if err != nil { + return fmt.Errorf("could not insert collection: %w", err) + } + + return nil +} + +func (c *Collections) Store(collection *flow.Collection) error { + return operation.RetryOnConflictTx(c.db, transaction.Update, func(ttx *transaction.Tx) error { + light := collection.Light() + err := transaction.WithTx(operation.SkipDuplicates(operation.InsertCollection(&light)))(ttx) + if err != nil { + return fmt.Errorf("could not insert collection: %w", err) + } + + for _, tx := range collection.Transactions { + err = c.transactions.storeTx(tx)(ttx) + if err != nil { + return fmt.Errorf("could not insert transaction: %w", err) + } + } + + return nil + }) +} + +func (c *Collections) ByID(colID flow.Identifier) (*flow.Collection, error) { + var ( + light flow.LightCollection + collection flow.Collection + ) + + err := c.db.View(func(btx *badger.Txn) error { + err := operation.RetrieveCollection(colID, &light)(btx) + if err != nil { + return fmt.Errorf("could not retrieve collection: %w", err) + } + + for _, txID := range light.Transactions { + tx, err := c.transactions.ByID(txID) + if err != nil { + return fmt.Errorf("could not retrieve transaction: %w", err) + } + + collection.Transactions = append(collection.Transactions, tx) + } + + return nil + }) + if err != nil { + return nil, err + } + + return &collection, nil +} + +func (c *Collections) LightByID(colID flow.Identifier) (*flow.LightCollection, error) { + var collection flow.LightCollection + + err := c.db.View(func(tx *badger.Txn) error { + err := operation.RetrieveCollection(colID, &collection)(tx) + if err != nil { + return fmt.Errorf("could not retrieve collection: %w", err) + } + + return nil + }) + if err != nil { + return nil, err + } + + return &collection, nil +} + +func (c *Collections) Remove(colID flow.Identifier) error { + return operation.RetryOnConflict(c.db.Update, func(btx *badger.Txn) error { + err := operation.RemoveCollection(colID)(btx) + if err != nil { + return fmt.Errorf("could not remove collection: %w", err) + } + return nil + }) +} + +func (c *Collections) StoreLightAndIndexByTransaction(collection *flow.LightCollection) error { + return operation.RetryOnConflict(c.db.Update, func(tx *badger.Txn) error { + err := operation.InsertCollection(collection)(tx) + if err != nil { + return fmt.Errorf("could not insert collection: %w", err) + } + + for _, txID := range collection.Transactions { + err = operation.IndexCollectionByTransaction(txID, collection.ID())(tx) + if errors.Is(err, storage.ErrAlreadyExists) { + continue + } + if err != nil { + return fmt.Errorf("could not insert transaction ID: %w", err) + } + } + + return nil + }) +} + +func (c *Collections) LightByTransactionID(txID flow.Identifier) (*flow.LightCollection, error) { + var collection flow.LightCollection + err := c.db.View(func(tx *badger.Txn) error { + collID := &flow.Identifier{} + err := operation.RetrieveCollectionID(txID, collID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve collection id: %w", err) + } + + err = operation.RetrieveCollection(*collID, &collection)(tx) + if err != nil { + return fmt.Errorf("could not retrieve collection: %w", err) + } + + return nil + }) + if err != nil { + return nil, err + } + + return &collection, nil +} diff --git a/storage/pebble/collections_test.go b/storage/pebble/collections_test.go new file mode 100644 index 00000000000..f6a8db73729 --- /dev/null +++ b/storage/pebble/collections_test.go @@ -0,0 +1,87 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestCollections(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + metrics := metrics.NewNoopCollector() + transactions := bstorage.NewTransactions(metrics, db) + collections := bstorage.NewCollections(db, transactions) + + // create a light collection with three transactions + expected := unittest.CollectionFixture(3).Light() + + // store the light collection and the transaction index + err := collections.StoreLightAndIndexByTransaction(&expected) + require.Nil(t, err) + + // retrieve the light collection by collection id + actual, err := collections.LightByID(expected.ID()) + require.Nil(t, err) + + // check if the light collection was indeed persisted + assert.Equal(t, &expected, actual) + + expectedID := expected.ID() + + // retrieve the collection light id by each of its transaction id + for _, txID := range expected.Transactions { + collLight, err := collections.LightByTransactionID(txID) + actualID := collLight.ID() + // check that the collection id can indeed be retrieved by transaction id + require.Nil(t, err) + assert.Equal(t, expectedID, actualID) + } + + }) +} + +func TestCollections_IndexDuplicateTx(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + transactions := bstorage.NewTransactions(metrics, db) + collections := bstorage.NewCollections(db, transactions) + + // create two collections which share 1 transaction + col1 := unittest.CollectionFixture(2) + col2 := unittest.CollectionFixture(1) + dupTx := col1.Transactions[0] // the duplicated transaction + col2Tx := col2.Transactions[0] // transaction that's only in col2 + col2.Transactions = append(col2.Transactions, dupTx) + + // insert col1 + col1Light := col1.Light() + err := collections.StoreLightAndIndexByTransaction(&col1Light) + require.NoError(t, err) + + // insert col2 + col2Light := col2.Light() + err = collections.StoreLightAndIndexByTransaction(&col2Light) + require.NoError(t, err) + + // should be able to retrieve col2 by ID + gotLightByCol2ID, err := collections.LightByID(col2.ID()) + require.NoError(t, err) + assert.Equal(t, &col2Light, gotLightByCol2ID) + + // should be able to retrieve col2 by the transaction which only appears in col2 + _, err = collections.LightByTransactionID(col2Tx.ID()) + require.NoError(t, err) + + // col1 (not col2) should be indexed by the shared transaction (since col1 was inserted first) + gotLightByDupTxID, err := collections.LightByTransactionID(dupTx.ID()) + require.NoError(t, err) + assert.Equal(t, &col1Light, gotLightByDupTxID) + }) +} diff --git a/storage/pebble/commit_test.go b/storage/pebble/commit_test.go new file mode 100644 index 00000000000..25527c31c61 --- /dev/null +++ b/storage/pebble/commit_test.go @@ -0,0 +1,43 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +// TestCommitsStoreAndRetrieve tests that a commit can be stored, retrieved and attempted to be stored again without an error +func TestCommitsStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewCommits(metrics, db) + + // attempt to get a invalid commit + _, err := store.ByBlockID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // store a commit in db + blockID := unittest.IdentifierFixture() + expected := unittest.StateCommitmentFixture() + err = store.Store(blockID, expected) + require.NoError(t, err) + + // retrieve the commit by ID + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + assert.Equal(t, expected, actual) + + // re-insert the commit - should be idempotent + err = store.Store(blockID, expected) + require.NoError(t, err) + }) +} diff --git a/storage/pebble/commits.go b/storage/pebble/commits.go new file mode 100644 index 00000000000..11a4e4aa8e2 --- /dev/null +++ b/storage/pebble/commits.go @@ -0,0 +1,89 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type Commits struct { + db *badger.DB + cache *Cache[flow.Identifier, flow.StateCommitment] +} + +func NewCommits(collector module.CacheMetrics, db *badger.DB) *Commits { + + store := func(blockID flow.Identifier, commit flow.StateCommitment) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.IndexStateCommitment(blockID, commit))) + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (flow.StateCommitment, error) { + return func(tx *badger.Txn) (flow.StateCommitment, error) { + var commit flow.StateCommitment + err := operation.LookupStateCommitment(blockID, &commit)(tx) + return commit, err + } + } + + c := &Commits{ + db: db, + cache: newCache[flow.Identifier, flow.StateCommitment](collector, metrics.ResourceCommit, + withLimit[flow.Identifier, flow.StateCommitment](1000), + withStore(store), + withRetrieve(retrieve), + ), + } + + return c +} + +func (c *Commits) storeTx(blockID flow.Identifier, commit flow.StateCommitment) func(*transaction.Tx) error { + return c.cache.PutTx(blockID, commit) +} + +func (c *Commits) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn) (flow.StateCommitment, error) { + return func(tx *badger.Txn) (flow.StateCommitment, error) { + val, err := c.cache.Get(blockID)(tx) + if err != nil { + return flow.DummyStateCommitment, err + } + return val, nil + } +} + +func (c *Commits) Store(blockID flow.Identifier, commit flow.StateCommitment) error { + return operation.RetryOnConflictTx(c.db, transaction.Update, c.storeTx(blockID, commit)) +} + +// BatchStore stores Commit keyed by blockID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (c *Commits) BatchStore(blockID flow.Identifier, commit flow.StateCommitment, batch storage.BatchStorage) error { + // we can't cache while using batches, as it's unknown at this point when, and if + // the batch will be committed. Cache will be populated on read however. + writeBatch := batch.GetWriter() + return operation.BatchIndexStateCommitment(blockID, commit)(writeBatch) +} + +func (c *Commits) ByBlockID(blockID flow.Identifier) (flow.StateCommitment, error) { + tx := c.db.NewTransaction(false) + defer tx.Discard() + return c.retrieveTx(blockID)(tx) +} + +func (c *Commits) RemoveByBlockID(blockID flow.Identifier) error { + return c.db.Update(operation.SkipNonExist(operation.RemoveStateCommitment(blockID))) +} + +// BatchRemoveByBlockID removes Commit keyed by blockID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (c *Commits) BatchRemoveByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return operation.BatchRemoveStateCommitment(blockID)(writeBatch) +} diff --git a/storage/pebble/common.go b/storage/pebble/common.go new file mode 100644 index 00000000000..77c6c5e7296 --- /dev/null +++ b/storage/pebble/common.go @@ -0,0 +1,21 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/storage" +) + +func handleError(err error, t interface{}) error { + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return storage.ErrNotFound + } + + return fmt.Errorf("could not retrieve %T: %w", t, err) + } + return nil +} diff --git a/storage/pebble/computation_result.go b/storage/pebble/computation_result.go new file mode 100644 index 00000000000..8338884334a --- /dev/null +++ b/storage/pebble/computation_result.go @@ -0,0 +1,49 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" +) + +type ComputationResultUploadStatus struct { + db *badger.DB +} + +func NewComputationResultUploadStatus(db *badger.DB) *ComputationResultUploadStatus { + return &ComputationResultUploadStatus{ + db: db, + } +} + +func (c *ComputationResultUploadStatus) Upsert(blockID flow.Identifier, + wasUploadCompleted bool) error { + return operation.RetryOnConflict(c.db.Update, func(btx *badger.Txn) error { + return operation.UpsertComputationResultUploadStatus(blockID, wasUploadCompleted)(btx) + }) +} + +func (c *ComputationResultUploadStatus) GetIDsByUploadStatus(targetUploadStatus bool) ([]flow.Identifier, error) { + ids := make([]flow.Identifier, 0) + err := c.db.View(operation.GetBlockIDsByStatus(&ids, targetUploadStatus)) + return ids, err +} + +func (c *ComputationResultUploadStatus) ByID(computationResultID flow.Identifier) (bool, error) { + var ret bool + err := c.db.View(func(btx *badger.Txn) error { + return operation.GetComputationResultUploadStatus(computationResultID, &ret)(btx) + }) + if err != nil { + return false, err + } + + return ret, nil +} + +func (c *ComputationResultUploadStatus) Remove(computationResultID flow.Identifier) error { + return operation.RetryOnConflict(c.db.Update, func(btx *badger.Txn) error { + return operation.RemoveComputationResultUploadStatus(computationResultID)(btx) + }) +} diff --git a/storage/pebble/computation_result_test.go b/storage/pebble/computation_result_test.go new file mode 100644 index 00000000000..6575611632c --- /dev/null +++ b/storage/pebble/computation_result_test.go @@ -0,0 +1,109 @@ +package badger_test + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/engine/execution" + "github.com/onflow/flow-go/engine/execution/testutil" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestUpsertAndRetrieveComputationResult(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := testutil.ComputationResultFixture(t) + crStorage := bstorage.NewComputationResultUploadStatus(db) + crId := expected.ExecutableBlock.ID() + + // True case - upsert + testUploadStatus := true + err := crStorage.Upsert(crId, testUploadStatus) + require.NoError(t, err) + + actualUploadStatus, err := crStorage.ByID(crId) + require.NoError(t, err) + + assert.Equal(t, testUploadStatus, actualUploadStatus) + + // False case - update + testUploadStatus = false + err = crStorage.Upsert(crId, testUploadStatus) + require.NoError(t, err) + + actualUploadStatus, err = crStorage.ByID(crId) + require.NoError(t, err) + + assert.Equal(t, testUploadStatus, actualUploadStatus) + }) +} + +func TestRemoveComputationResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("Remove ComputationResult", func(t *testing.T) { + expected := testutil.ComputationResultFixture(t) + crId := expected.ExecutableBlock.ID() + crStorage := bstorage.NewComputationResultUploadStatus(db) + + testUploadStatus := true + err := crStorage.Upsert(crId, testUploadStatus) + require.NoError(t, err) + + _, err = crStorage.ByID(crId) + require.NoError(t, err) + + err = crStorage.Remove(crId) + require.NoError(t, err) + + _, err = crStorage.ByID(crId) + assert.Error(t, err) + }) + }) +} + +func TestListComputationResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("List all ComputationResult with given status", func(t *testing.T) { + expected := [...]*execution.ComputationResult{ + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), + } + crStorage := bstorage.NewComputationResultUploadStatus(db) + + // Store a list of ComputationResult instances first + expectedIDs := make(map[string]bool, 0) + for _, cr := range expected { + crId := cr.ExecutableBlock.ID() + expectedIDs[crId.String()] = true + err := crStorage.Upsert(crId, true) + require.NoError(t, err) + } + // Add in entries with non-targeted status + unexpected := [...]*execution.ComputationResult{ + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), + } + for _, cr := range unexpected { + crId := cr.ExecutableBlock.ID() + err := crStorage.Upsert(crId, false) + require.NoError(t, err) + } + + // Get the list of IDs for stored instances + crIDs, err := crStorage.GetIDsByUploadStatus(true) + require.NoError(t, err) + + crIDsStrMap := make(map[string]bool, 0) + for _, crID := range crIDs { + crIDsStrMap[crID.String()] = true + } + + assert.True(t, reflect.DeepEqual(crIDsStrMap, expectedIDs)) + }) + }) +} diff --git a/storage/pebble/consumer_progress.go b/storage/pebble/consumer_progress.go new file mode 100644 index 00000000000..52855dd60b1 --- /dev/null +++ b/storage/pebble/consumer_progress.go @@ -0,0 +1,50 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/storage/badger/operation" +) + +type ConsumerProgress struct { + db *badger.DB + consumer string // to distinguish the consume progress between different consumers +} + +func NewConsumerProgress(db *badger.DB, consumer string) *ConsumerProgress { + return &ConsumerProgress{ + db: db, + consumer: consumer, + } +} + +func (cp *ConsumerProgress) ProcessedIndex() (uint64, error) { + var processed uint64 + err := cp.db.View(operation.RetrieveProcessedIndex(cp.consumer, &processed)) + if err != nil { + return 0, fmt.Errorf("failed to retrieve processed index: %w", err) + } + return processed, nil +} + +// InitProcessedIndex insert the default processed index to the storage layer, can only be done once. +// initialize for the second time will return storage.ErrAlreadyExists +func (cp *ConsumerProgress) InitProcessedIndex(defaultIndex uint64) error { + err := operation.RetryOnConflict(cp.db.Update, operation.InsertProcessedIndex(cp.consumer, defaultIndex)) + if err != nil { + return fmt.Errorf("could not update processed index: %w", err) + } + + return nil +} + +func (cp *ConsumerProgress) SetProcessedIndex(processed uint64) error { + err := operation.RetryOnConflict(cp.db.Update, operation.SetProcessedIndex(cp.consumer, processed)) + if err != nil { + return fmt.Errorf("could not update processed index: %w", err) + } + + return nil +} diff --git a/storage/pebble/dkg_state.go b/storage/pebble/dkg_state.go new file mode 100644 index 00000000000..73e2b3e8133 --- /dev/null +++ b/storage/pebble/dkg_state.go @@ -0,0 +1,176 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/encodable" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// DKGState stores state information about in-progress and completed DKGs, including +// computed keys. Must be instantiated using secrets database. +type DKGState struct { + db *badger.DB + keyCache *Cache[uint64, *encodable.RandomBeaconPrivKey] +} + +// NewDKGState returns the DKGState implementation backed by Badger DB. +func NewDKGState(collector module.CacheMetrics, db *badger.DB) (*DKGState, error) { + err := operation.EnsureSecretDB(db) + if err != nil { + return nil, fmt.Errorf("cannot instantiate dkg state storage in non-secret db: %w", err) + } + + storeKey := func(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*transaction.Tx) error { + return transaction.WithTx(operation.InsertMyBeaconPrivateKey(epochCounter, info)) + } + + retrieveKey := func(epochCounter uint64) func(*badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + return func(tx *badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + var info encodable.RandomBeaconPrivKey + err := operation.RetrieveMyBeaconPrivateKey(epochCounter, &info)(tx) + return &info, err + } + } + + cache := newCache[uint64, *encodable.RandomBeaconPrivKey](collector, metrics.ResourceBeaconKey, + withLimit[uint64, *encodable.RandomBeaconPrivKey](10), + withStore(storeKey), + withRetrieve(retrieveKey), + ) + + dkgState := &DKGState{ + db: db, + keyCache: cache, + } + + return dkgState, nil +} + +func (ds *DKGState) storeKeyTx(epochCounter uint64, key *encodable.RandomBeaconPrivKey) func(tx *transaction.Tx) error { + return ds.keyCache.PutTx(epochCounter, key) +} + +func (ds *DKGState) retrieveKeyTx(epochCounter uint64) func(tx *badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + return func(tx *badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + val, err := ds.keyCache.Get(epochCounter)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +// InsertMyBeaconPrivateKey stores the random beacon private key for an epoch. +// +// CAUTION: these keys are stored before they are validated against the +// canonical key vector and may not be valid for use in signing. Use SafeBeaconKeys +// to guarantee only keys safe for signing are returned +func (ds *DKGState) InsertMyBeaconPrivateKey(epochCounter uint64, key crypto.PrivateKey) error { + if key == nil { + return fmt.Errorf("will not store nil beacon key") + } + encodableKey := &encodable.RandomBeaconPrivKey{PrivateKey: key} + return operation.RetryOnConflictTx(ds.db, transaction.Update, ds.storeKeyTx(epochCounter, encodableKey)) +} + +// RetrieveMyBeaconPrivateKey retrieves the random beacon private key for an epoch. +// +// CAUTION: these keys are stored before they are validated against the +// canonical key vector and may not be valid for use in signing. Use SafeBeaconKeys +// to guarantee only keys safe for signing are returned +func (ds *DKGState) RetrieveMyBeaconPrivateKey(epochCounter uint64) (crypto.PrivateKey, error) { + tx := ds.db.NewTransaction(false) + defer tx.Discard() + encodableKey, err := ds.retrieveKeyTx(epochCounter)(tx) + if err != nil { + return nil, err + } + return encodableKey.PrivateKey, nil +} + +// SetDKGStarted sets the flag indicating the DKG has started for the given epoch. +func (ds *DKGState) SetDKGStarted(epochCounter uint64) error { + return ds.db.Update(operation.InsertDKGStartedForEpoch(epochCounter)) +} + +// GetDKGStarted checks whether the DKG has been started for the given epoch. +func (ds *DKGState) GetDKGStarted(epochCounter uint64) (bool, error) { + var started bool + err := ds.db.View(operation.RetrieveDKGStartedForEpoch(epochCounter, &started)) + return started, err +} + +// SetDKGEndState stores that the DKG has ended, and its end state. +func (ds *DKGState) SetDKGEndState(epochCounter uint64, endState flow.DKGEndState) error { + return ds.db.Update(operation.InsertDKGEndStateForEpoch(epochCounter, endState)) +} + +// GetDKGEndState retrieves the DKG end state for the epoch. +func (ds *DKGState) GetDKGEndState(epochCounter uint64) (flow.DKGEndState, error) { + var endState flow.DKGEndState + err := ds.db.Update(operation.RetrieveDKGEndStateForEpoch(epochCounter, &endState)) + return endState, err +} + +// SafeBeaconPrivateKeys is the safe beacon key storage backed by Badger DB. +type SafeBeaconPrivateKeys struct { + state *DKGState +} + +// NewSafeBeaconPrivateKeys returns a safe beacon key storage backed by Badger DB. +func NewSafeBeaconPrivateKeys(state *DKGState) *SafeBeaconPrivateKeys { + return &SafeBeaconPrivateKeys{state: state} +} + +// RetrieveMyBeaconPrivateKey retrieves my beacon private key for the given +// epoch, only if my key has been confirmed valid and safe for use. +// +// Returns: +// - (key, true, nil) if the key is present and confirmed valid +// - (nil, false, nil) if the key has been marked invalid or unavailable +// -> no beacon key will ever be available for the epoch in this case +// - (nil, false, storage.ErrNotFound) if the DKG has not ended +// - (nil, false, error) for any unexpected exception +func (keys *SafeBeaconPrivateKeys) RetrieveMyBeaconPrivateKey(epochCounter uint64) (key crypto.PrivateKey, safe bool, err error) { + err = keys.state.db.View(func(txn *badger.Txn) error { + + // retrieve the end state + var endState flow.DKGEndState + err = operation.RetrieveDKGEndStateForEpoch(epochCounter, &endState)(txn) + if err != nil { + key = nil + safe = false + return err // storage.ErrNotFound or exception + } + + // for any end state besides success, the key is not safe + if endState != flow.DKGEndStateSuccess { + key = nil + safe = false + return nil + } + + // retrieve the key - any storage error (including not found) is an exception + var encodableKey *encodable.RandomBeaconPrivKey + encodableKey, err = keys.state.retrieveKeyTx(epochCounter)(txn) + if err != nil { + key = nil + safe = false + return fmt.Errorf("[unexpected] could not retrieve beacon key for epoch %d with successful DKG: %v", epochCounter, err) + } + + // return the key only for successful end state + safe = true + key = encodableKey.PrivateKey + return nil + }) + return +} diff --git a/storage/pebble/dkg_state_test.go b/storage/pebble/dkg_state_test.go new file mode 100644 index 00000000000..5643b064d22 --- /dev/null +++ b/storage/pebble/dkg_state_test.go @@ -0,0 +1,232 @@ +package badger_test + +import ( + "errors" + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestDKGState_DKGStarted(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitSecret, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store, err := bstorage.NewDKGState(metrics, db) + require.NoError(t, err) + + epochCounter := rand.Uint64() + + // check dkg-started flag for non-existent epoch + t.Run("DKGStarted should default to false", func(t *testing.T) { + started, err := store.GetDKGStarted(rand.Uint64()) + assert.NoError(t, err) + assert.False(t, started) + }) + + // store dkg-started flag for epoch + t.Run("should be able to set DKGStarted", func(t *testing.T) { + err = store.SetDKGStarted(epochCounter) + assert.NoError(t, err) + }) + + // retrieve flag for epoch + t.Run("should be able to read DKGStarted", func(t *testing.T) { + started, err := store.GetDKGStarted(epochCounter) + assert.NoError(t, err) + assert.True(t, started) + }) + }) +} + +func TestDKGState_BeaconKeys(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitSecret, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store, err := bstorage.NewDKGState(metrics, db) + require.NoError(t, err) + + epochCounter := rand.Uint64() + + // attempt to get a non-existent key + t.Run("should error if retrieving non-existent key", func(t *testing.T) { + _, err = store.RetrieveMyBeaconPrivateKey(epochCounter) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) + + // attempt to store a nil key should fail - use DKGState.SetEndState(flow.DKGEndStateNoKey) + t.Run("should fail to store a nil key instead)", func(t *testing.T) { + err = store.InsertMyBeaconPrivateKey(epochCounter, nil) + assert.Error(t, err) + }) + + // store a key in db + expected := unittest.RandomBeaconPriv() + t.Run("should be able to store and read a key", func(t *testing.T) { + err = store.InsertMyBeaconPrivateKey(epochCounter, expected) + require.NoError(t, err) + }) + + // retrieve the key by epoch counter + t.Run("should be able to retrieve stored key", func(t *testing.T) { + actual, err := store.RetrieveMyBeaconPrivateKey(epochCounter) + require.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + // test storing same key + t.Run("should fail to store a key twice", func(t *testing.T) { + err = store.InsertMyBeaconPrivateKey(epochCounter, expected) + require.True(t, errors.Is(err, storage.ErrAlreadyExists)) + }) + }) +} + +func TestDKGState_EndState(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitSecret, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store, err := bstorage.NewDKGState(metrics, db) + require.NoError(t, err) + + epochCounter := rand.Uint64() + endState := flow.DKGEndStateNoKey + + t.Run("should be able to store an end state", func(t *testing.T) { + err = store.SetDKGEndState(epochCounter, endState) + require.NoError(t, err) + }) + + t.Run("should be able to read an end state", func(t *testing.T) { + readEndState, err := store.GetDKGEndState(epochCounter) + require.NoError(t, err) + assert.Equal(t, endState, readEndState) + }) + }) +} + +func TestSafeBeaconPrivateKeys(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitSecret, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + dkgState, err := bstorage.NewDKGState(metrics, db) + require.NoError(t, err) + safeKeys := bstorage.NewSafeBeaconPrivateKeys(dkgState) + + t.Run("non-existent key -> should return ErrNotFound", func(t *testing.T) { + epochCounter := rand.Uint64() + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) + + t.Run("existent key, non-existent end state -> should return ErrNotFound", func(t *testing.T) { + epochCounter := rand.Uint64() + + // store a key + expected := unittest.RandomBeaconPriv().PrivateKey + err := dkgState.InsertMyBeaconPrivateKey(epochCounter, expected) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) + + t.Run("existent key, unsuccessful end state -> not safe", func(t *testing.T) { + epochCounter := rand.Uint64() + + // store a key + expected := unittest.RandomBeaconPriv().PrivateKey + err := dkgState.InsertMyBeaconPrivateKey(epochCounter, expected) + assert.NoError(t, err) + // mark dkg unsuccessful + err = dkgState.SetDKGEndState(epochCounter, flow.DKGEndStateInconsistentKey) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.NoError(t, err) + }) + + t.Run("existent key, inconsistent key end state -> not safe", func(t *testing.T) { + epochCounter := rand.Uint64() + + // store a key + expected := unittest.RandomBeaconPriv().PrivateKey + err := dkgState.InsertMyBeaconPrivateKey(epochCounter, expected) + assert.NoError(t, err) + // mark dkg result as inconsistent + err = dkgState.SetDKGEndState(epochCounter, flow.DKGEndStateInconsistentKey) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.NoError(t, err) + }) + + t.Run("non-existent key, no key end state -> not safe", func(t *testing.T) { + epochCounter := rand.Uint64() + + // mark dkg result as no key + err = dkgState.SetDKGEndState(epochCounter, flow.DKGEndStateNoKey) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.NoError(t, err) + }) + + t.Run("existent key, successful end state -> safe", func(t *testing.T) { + epochCounter := rand.Uint64() + + // store a key + expected := unittest.RandomBeaconPriv().PrivateKey + err := dkgState.InsertMyBeaconPrivateKey(epochCounter, expected) + assert.NoError(t, err) + // mark dkg successful + err = dkgState.SetDKGEndState(epochCounter, flow.DKGEndStateSuccess) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.NotNil(t, key) + assert.True(t, expected.Equals(key)) + assert.True(t, safe) + assert.NoError(t, err) + }) + + t.Run("non-existent key, successful end state -> exception!", func(t *testing.T) { + epochCounter := rand.Uint64() + + // mark dkg successful + err = dkgState.SetDKGEndState(epochCounter, flow.DKGEndStateSuccess) + assert.NoError(t, err) + + key, safe, err := safeKeys.RetrieveMyBeaconPrivateKey(epochCounter) + assert.Nil(t, key) + assert.False(t, safe) + assert.Error(t, err) + assert.NotErrorIs(t, err, storage.ErrNotFound) + }) + + }) +} + +// TestSecretDBRequirement tests that the DKGState constructor will return an +// error if instantiated using a database not marked with the correct type. +func TestSecretDBRequirement(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + _, err := bstorage.NewDKGState(metrics, db) + require.Error(t, err) + }) +} diff --git a/storage/pebble/epoch_commits.go b/storage/pebble/epoch_commits.go new file mode 100644 index 00000000000..20dadaccdba --- /dev/null +++ b/storage/pebble/epoch_commits.go @@ -0,0 +1,69 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type EpochCommits struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.EpochCommit] +} + +func NewEpochCommits(collector module.CacheMetrics, db *badger.DB) *EpochCommits { + + store := func(id flow.Identifier, commit *flow.EpochCommit) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertEpochCommit(id, commit))) + } + + retrieve := func(id flow.Identifier) func(*badger.Txn) (*flow.EpochCommit, error) { + return func(tx *badger.Txn) (*flow.EpochCommit, error) { + var commit flow.EpochCommit + err := operation.RetrieveEpochCommit(id, &commit)(tx) + return &commit, err + } + } + + ec := &EpochCommits{ + db: db, + cache: newCache[flow.Identifier, *flow.EpochCommit](collector, metrics.ResourceEpochCommit, + withLimit[flow.Identifier, *flow.EpochCommit](4*flow.DefaultTransactionExpiry), + withStore(store), + withRetrieve(retrieve)), + } + + return ec +} + +func (ec *EpochCommits) StoreTx(commit *flow.EpochCommit) func(*transaction.Tx) error { + return ec.cache.PutTx(commit.ID(), commit) +} + +func (ec *EpochCommits) retrieveTx(commitID flow.Identifier) func(tx *badger.Txn) (*flow.EpochCommit, error) { + return func(tx *badger.Txn) (*flow.EpochCommit, error) { + val, err := ec.cache.Get(commitID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +// TODO: can we remove this method? Its not contained in the interface. +func (ec *EpochCommits) Store(commit *flow.EpochCommit) error { + return operation.RetryOnConflictTx(ec.db, transaction.Update, ec.StoreTx(commit)) +} + +// ByID will return the EpochCommit event by its ID. +// Error returns: +// * storage.ErrNotFound if no EpochCommit with the ID exists +func (ec *EpochCommits) ByID(commitID flow.Identifier) (*flow.EpochCommit, error) { + tx := ec.db.NewTransaction(false) + defer tx.Discard() + return ec.retrieveTx(commitID)(tx) +} diff --git a/storage/pebble/epoch_commits_test.go b/storage/pebble/epoch_commits_test.go new file mode 100644 index 00000000000..aacbf81f7b9 --- /dev/null +++ b/storage/pebble/epoch_commits_test.go @@ -0,0 +1,42 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +// TestEpochCommitStoreAndRetrieve tests that a commit can be stored, retrieved and attempted to be stored again without an error +func TestEpochCommitStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewEpochCommits(metrics, db) + + // attempt to get a invalid commit + _, err := store.ByID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // store a commit in db + expected := unittest.EpochCommitFixture() + err = store.Store(expected) + require.NoError(t, err) + + // retrieve the commit by ID + actual, err := store.ByID(expected.ID()) + require.NoError(t, err) + assert.Equal(t, expected, actual) + + // test storing same epoch commit + err = store.Store(expected) + require.NoError(t, err) + }) +} diff --git a/storage/pebble/epoch_setups.go b/storage/pebble/epoch_setups.go new file mode 100644 index 00000000000..24757067f8f --- /dev/null +++ b/storage/pebble/epoch_setups.go @@ -0,0 +1,65 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type EpochSetups struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.EpochSetup] +} + +// NewEpochSetups instantiates a new EpochSetups storage. +func NewEpochSetups(collector module.CacheMetrics, db *badger.DB) *EpochSetups { + + store := func(id flow.Identifier, setup *flow.EpochSetup) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertEpochSetup(id, setup))) + } + + retrieve := func(id flow.Identifier) func(*badger.Txn) (*flow.EpochSetup, error) { + return func(tx *badger.Txn) (*flow.EpochSetup, error) { + var setup flow.EpochSetup + err := operation.RetrieveEpochSetup(id, &setup)(tx) + return &setup, err + } + } + + es := &EpochSetups{ + db: db, + cache: newCache[flow.Identifier, *flow.EpochSetup](collector, metrics.ResourceEpochSetup, + withLimit[flow.Identifier, *flow.EpochSetup](4*flow.DefaultTransactionExpiry), + withStore(store), + withRetrieve(retrieve)), + } + + return es +} + +func (es *EpochSetups) StoreTx(setup *flow.EpochSetup) func(tx *transaction.Tx) error { + return es.cache.PutTx(setup.ID(), setup) +} + +func (es *EpochSetups) retrieveTx(setupID flow.Identifier) func(tx *badger.Txn) (*flow.EpochSetup, error) { + return func(tx *badger.Txn) (*flow.EpochSetup, error) { + val, err := es.cache.Get(setupID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +// ByID will return the EpochSetup event by its ID. +// Error returns: +// * storage.ErrNotFound if no EpochSetup with the ID exists +func (es *EpochSetups) ByID(setupID flow.Identifier) (*flow.EpochSetup, error) { + tx := es.db.NewTransaction(false) + defer tx.Discard() + return es.retrieveTx(setupID)(tx) +} diff --git a/storage/pebble/epoch_setups_test.go b/storage/pebble/epoch_setups_test.go new file mode 100644 index 00000000000..fae4b153c1c --- /dev/null +++ b/storage/pebble/epoch_setups_test.go @@ -0,0 +1,44 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// TestEpochSetupStoreAndRetrieve tests that a setup can be stored, retrieved and attempted to be stored again without an error +func TestEpochSetupStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewEpochSetups(metrics, db) + + // attempt to get a setup that doesn't exist + _, err := store.ByID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // store a setup in db + expected := unittest.EpochSetupFixture() + err = operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(expected)) + require.NoError(t, err) + + // retrieve the setup by ID + actual, err := store.ByID(expected.ID()) + require.NoError(t, err) + assert.Equal(t, expected, actual) + + // test storing same epoch setup + err = operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(expected)) + require.NoError(t, err) + }) +} diff --git a/storage/pebble/epoch_statuses.go b/storage/pebble/epoch_statuses.go new file mode 100644 index 00000000000..2d64fcfea8f --- /dev/null +++ b/storage/pebble/epoch_statuses.go @@ -0,0 +1,65 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type EpochStatuses struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.EpochStatus] +} + +// NewEpochStatuses ... +func NewEpochStatuses(collector module.CacheMetrics, db *badger.DB) *EpochStatuses { + + store := func(blockID flow.Identifier, status *flow.EpochStatus) func(*transaction.Tx) error { + return transaction.WithTx(operation.InsertEpochStatus(blockID, status)) + } + + retrieve := func(blockID flow.Identifier) func(*badger.Txn) (*flow.EpochStatus, error) { + return func(tx *badger.Txn) (*flow.EpochStatus, error) { + var status flow.EpochStatus + err := operation.RetrieveEpochStatus(blockID, &status)(tx) + return &status, err + } + } + + es := &EpochStatuses{ + db: db, + cache: newCache[flow.Identifier, *flow.EpochStatus](collector, metrics.ResourceEpochStatus, + withLimit[flow.Identifier, *flow.EpochStatus](4*flow.DefaultTransactionExpiry), + withStore(store), + withRetrieve(retrieve)), + } + + return es +} + +func (es *EpochStatuses) StoreTx(blockID flow.Identifier, status *flow.EpochStatus) func(tx *transaction.Tx) error { + return es.cache.PutTx(blockID, status) +} + +func (es *EpochStatuses) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn) (*flow.EpochStatus, error) { + return func(tx *badger.Txn) (*flow.EpochStatus, error) { + val, err := es.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +// ByBlockID will return the epoch status for the given block +// Error returns: +// * storage.ErrNotFound if EpochStatus for the block does not exist +func (es *EpochStatuses) ByBlockID(blockID flow.Identifier) (*flow.EpochStatus, error) { + tx := es.db.NewTransaction(false) + defer tx.Discard() + return es.retrieveTx(blockID)(tx) +} diff --git a/storage/pebble/epoch_statuses_test.go b/storage/pebble/epoch_statuses_test.go new file mode 100644 index 00000000000..ce560bee9d2 --- /dev/null +++ b/storage/pebble/epoch_statuses_test.go @@ -0,0 +1,40 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +func TestEpochStatusesStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewEpochStatuses(metrics, db) + + blockID := unittest.IdentifierFixture() + expected := unittest.EpochStatusFixture() + + _, err := store.ByBlockID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + // store epoch status + err = operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(blockID, expected)) + require.NoError(t, err) + + // retreive status + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/events.go b/storage/pebble/events.go new file mode 100644 index 00000000000..ca7cb5105ec --- /dev/null +++ b/storage/pebble/events.go @@ -0,0 +1,227 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +type Events struct { + db *badger.DB + cache *Cache[flow.Identifier, []flow.Event] +} + +func NewEvents(collector module.CacheMetrics, db *badger.DB) *Events { + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) ([]flow.Event, error) { + var events []flow.Event + return func(tx *badger.Txn) ([]flow.Event, error) { + err := operation.LookupEventsByBlockID(blockID, &events)(tx) + return events, handleError(err, flow.Event{}) + } + } + + return &Events{ + db: db, + cache: newCache[flow.Identifier, []flow.Event](collector, metrics.ResourceEvents, + withStore(noopStore[flow.Identifier, []flow.Event]), + withRetrieve(retrieve)), + } +} + +// BatchStore stores events keyed by a blockID in provided batch +// No errors are expected during normal operation, but it may return generic error +// if badger fails to process request +func (e *Events) BatchStore(blockID flow.Identifier, blockEvents []flow.EventsList, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + + // pre-allocating and indexing slice is faster than appending + sliceSize := 0 + for _, b := range blockEvents { + sliceSize += len(b) + } + + combinedEvents := make([]flow.Event, sliceSize) + + eventIndex := 0 + + for _, events := range blockEvents { + for _, event := range events { + err := operation.BatchInsertEvent(blockID, event)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch insert event: %w", err) + } + combinedEvents[eventIndex] = event + eventIndex++ + } + } + + callback := func() { + e.cache.Insert(blockID, combinedEvents) + } + batch.OnSucceed(callback) + return nil +} + +// Store will store events for the given block ID +func (e *Events) Store(blockID flow.Identifier, blockEvents []flow.EventsList) error { + batch := NewBatch(e.db) + + err := e.BatchStore(blockID, blockEvents, batch) + if err != nil { + return err + } + + err = batch.Flush() + if err != nil { + return fmt.Errorf("cannot flush batch: %w", err) + } + + return nil +} + +// ByBlockID returns the events for the given block ID +// Note: This method will return an empty slice and no error if no entries for the blockID are found +func (e *Events) ByBlockID(blockID flow.Identifier) ([]flow.Event, error) { + tx := e.db.NewTransaction(false) + defer tx.Discard() + val, err := e.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil +} + +// ByBlockIDTransactionID returns the events for the given block ID and transaction ID +// Note: This method will return an empty slice and no error if no entries for the blockID are found +func (e *Events) ByBlockIDTransactionID(blockID flow.Identifier, txID flow.Identifier) ([]flow.Event, error) { + events, err := e.ByBlockID(blockID) + if err != nil { + return nil, handleError(err, flow.Event{}) + } + + var matched []flow.Event + for _, event := range events { + if event.TransactionID == txID { + matched = append(matched, event) + } + } + return matched, nil +} + +// ByBlockIDTransactionIndex returns the events for the given block ID and transaction index +// Note: This method will return an empty slice and no error if no entries for the blockID are found +func (e *Events) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) ([]flow.Event, error) { + events, err := e.ByBlockID(blockID) + if err != nil { + return nil, handleError(err, flow.Event{}) + } + + var matched []flow.Event + for _, event := range events { + if event.TransactionIndex == txIndex { + matched = append(matched, event) + } + } + return matched, nil +} + +// ByBlockIDEventType returns the events for the given block ID and event type +// Note: This method will return an empty slice and no error if no entries for the blockID are found +func (e *Events) ByBlockIDEventType(blockID flow.Identifier, eventType flow.EventType) ([]flow.Event, error) { + events, err := e.ByBlockID(blockID) + if err != nil { + return nil, handleError(err, flow.Event{}) + } + + var matched []flow.Event + for _, event := range events { + if event.Type == eventType { + matched = append(matched, event) + } + } + return matched, nil +} + +// RemoveByBlockID removes events by block ID +func (e *Events) RemoveByBlockID(blockID flow.Identifier) error { + return e.db.Update(operation.RemoveEventsByBlockID(blockID)) +} + +// BatchRemoveByBlockID removes events keyed by a blockID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (e *Events) BatchRemoveByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return e.db.View(operation.BatchRemoveEventsByBlockID(blockID, writeBatch)) +} + +type ServiceEvents struct { + db *badger.DB + cache *Cache[flow.Identifier, []flow.Event] +} + +func NewServiceEvents(collector module.CacheMetrics, db *badger.DB) *ServiceEvents { + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) ([]flow.Event, error) { + var events []flow.Event + return func(tx *badger.Txn) ([]flow.Event, error) { + err := operation.LookupServiceEventsByBlockID(blockID, &events)(tx) + return events, handleError(err, flow.Event{}) + } + } + + return &ServiceEvents{ + db: db, + cache: newCache[flow.Identifier, []flow.Event](collector, metrics.ResourceEvents, + withStore(noopStore[flow.Identifier, []flow.Event]), + withRetrieve(retrieve)), + } +} + +// BatchStore stores service events keyed by a blockID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (e *ServiceEvents) BatchStore(blockID flow.Identifier, events []flow.Event, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + for _, event := range events { + err := operation.BatchInsertServiceEvent(blockID, event)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch insert service event: %w", err) + } + } + + callback := func() { + e.cache.Insert(blockID, events) + } + batch.OnSucceed(callback) + return nil +} + +// ByBlockID returns the events for the given block ID +func (e *ServiceEvents) ByBlockID(blockID flow.Identifier) ([]flow.Event, error) { + tx := e.db.NewTransaction(false) + defer tx.Discard() + val, err := e.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil +} + +// RemoveByBlockID removes service events by block ID +func (e *ServiceEvents) RemoveByBlockID(blockID flow.Identifier) error { + return e.db.Update(operation.RemoveServiceEventsByBlockID(blockID)) +} + +// BatchRemoveByBlockID removes service events keyed by a blockID in provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (e *ServiceEvents) BatchRemoveByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return e.db.View(operation.BatchRemoveServiceEventsByBlockID(blockID, writeBatch)) +} diff --git a/storage/pebble/events_test.go b/storage/pebble/events_test.go new file mode 100644 index 00000000000..cb0e956395c --- /dev/null +++ b/storage/pebble/events_test.go @@ -0,0 +1,123 @@ +package badger_test + +import ( + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/systemcontracts" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestEventStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewEvents(metrics, db) + + blockID := unittest.IdentifierFixture() + tx1ID := unittest.IdentifierFixture() + tx2ID := unittest.IdentifierFixture() + evt1_1 := unittest.EventFixture(flow.EventAccountCreated, 0, 0, tx1ID, 0) + evt1_2 := unittest.EventFixture(flow.EventAccountCreated, 1, 1, tx2ID, 0) + + evt2_1 := unittest.EventFixture(flow.EventAccountUpdated, 2, 2, tx2ID, 0) + + expected := []flow.EventsList{ + {evt1_1, evt1_2}, + {evt2_1}, + } + + batch := badgerstorage.NewBatch(db) + // store event + err := store.BatchStore(blockID, expected, batch) + require.NoError(t, err) + + err = batch.Flush() + require.NoError(t, err) + + // retrieve by blockID + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Len(t, actual, 3) + require.Contains(t, actual, evt1_1) + require.Contains(t, actual, evt1_2) + require.Contains(t, actual, evt2_1) + + // retrieve by blockID and event type + actual, err = store.ByBlockIDEventType(blockID, flow.EventAccountCreated) + require.NoError(t, err) + require.Len(t, actual, 2) + require.Contains(t, actual, evt1_1) + require.Contains(t, actual, evt1_2) + + actual, err = store.ByBlockIDEventType(blockID, flow.EventAccountUpdated) + require.NoError(t, err) + require.Len(t, actual, 1) + require.Contains(t, actual, evt2_1) + + events := systemcontracts.ServiceEventsForChain(flow.Emulator) + + actual, err = store.ByBlockIDEventType(blockID, events.EpochSetup.EventType()) + require.NoError(t, err) + require.Len(t, actual, 0) + + // retrieve by blockID and transaction id + actual, err = store.ByBlockIDTransactionID(blockID, tx1ID) + require.NoError(t, err) + require.Len(t, actual, 1) + require.Contains(t, actual, evt1_1) + + // retrieve by blockID and transaction index + actual, err = store.ByBlockIDTransactionIndex(blockID, 1) + require.NoError(t, err) + require.Len(t, actual, 1) + require.Contains(t, actual, evt1_2) + + // test loading from database + + newStore := badgerstorage.NewEvents(metrics, db) + actual, err = newStore.ByBlockID(blockID) + require.NoError(t, err) + require.Len(t, actual, 3) + require.Contains(t, actual, evt1_1) + require.Contains(t, actual, evt1_2) + require.Contains(t, actual, evt2_1) + }) +} + +func TestEventRetrieveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewEvents(metrics, db) + + blockID := unittest.IdentifierFixture() + txID := unittest.IdentifierFixture() + txIndex := rand.Uint32() + + // retrieve by blockID + events, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.True(t, len(events) == 0) + + // retrieve by blockID and event type + events, err = store.ByBlockIDEventType(blockID, flow.EventAccountCreated) + require.NoError(t, err) + require.True(t, len(events) == 0) + + // retrieve by blockID and transaction id + events, err = store.ByBlockIDTransactionID(blockID, txID) + require.NoError(t, err) + require.True(t, len(events) == 0) + + // retrieve by blockID and transaction id + events, err = store.ByBlockIDTransactionIndex(blockID, txIndex) + require.NoError(t, err) + require.True(t, len(events) == 0) + + }) +} diff --git a/storage/pebble/guarantees.go b/storage/pebble/guarantees.go new file mode 100644 index 00000000000..b7befd342b6 --- /dev/null +++ b/storage/pebble/guarantees.go @@ -0,0 +1,66 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// Guarantees implements persistent storage for collection guarantees. +type Guarantees struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.CollectionGuarantee] +} + +func NewGuarantees(collector module.CacheMetrics, db *badger.DB, cacheSize uint) *Guarantees { + + store := func(collID flow.Identifier, guarantee *flow.CollectionGuarantee) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertGuarantee(collID, guarantee))) + } + + retrieve := func(collID flow.Identifier) func(*badger.Txn) (*flow.CollectionGuarantee, error) { + var guarantee flow.CollectionGuarantee + return func(tx *badger.Txn) (*flow.CollectionGuarantee, error) { + err := operation.RetrieveGuarantee(collID, &guarantee)(tx) + return &guarantee, err + } + } + + g := &Guarantees{ + db: db, + cache: newCache[flow.Identifier, *flow.CollectionGuarantee](collector, metrics.ResourceGuarantee, + withLimit[flow.Identifier, *flow.CollectionGuarantee](cacheSize), + withStore(store), + withRetrieve(retrieve)), + } + + return g +} + +func (g *Guarantees) storeTx(guarantee *flow.CollectionGuarantee) func(*transaction.Tx) error { + return g.cache.PutTx(guarantee.ID(), guarantee) +} + +func (g *Guarantees) retrieveTx(collID flow.Identifier) func(*badger.Txn) (*flow.CollectionGuarantee, error) { + return func(tx *badger.Txn) (*flow.CollectionGuarantee, error) { + val, err := g.cache.Get(collID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +func (g *Guarantees) Store(guarantee *flow.CollectionGuarantee) error { + return operation.RetryOnConflictTx(g.db, transaction.Update, g.storeTx(guarantee)) +} + +func (g *Guarantees) ByCollectionID(collID flow.Identifier) (*flow.CollectionGuarantee, error) { + tx := g.db.NewTransaction(false) + defer tx.Discard() + return g.retrieveTx(collID)(tx) +} diff --git a/storage/pebble/guarantees_test.go b/storage/pebble/guarantees_test.go new file mode 100644 index 00000000000..778febfb49c --- /dev/null +++ b/storage/pebble/guarantees_test.go @@ -0,0 +1,38 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestGuaranteeStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewGuarantees(metrics, db, 1000) + + // abiturary guarantees + expected := unittest.CollectionGuaranteeFixture() + + // retrieve guarantee without stored + _, err := store.ByCollectionID(expected.ID()) + require.True(t, errors.Is(err, storage.ErrNotFound)) + + // store guarantee + err = store.Store(expected) + require.NoError(t, err) + + // retreive by coll idx + actual, err := store.ByCollectionID(expected.ID()) + require.NoError(t, err) + require.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/headers.go b/storage/pebble/headers.go new file mode 100644 index 00000000000..49574e5abc9 --- /dev/null +++ b/storage/pebble/headers.go @@ -0,0 +1,198 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// Headers implements a simple read-only header storage around a badger DB. +type Headers struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.Header] + heightCache *Cache[uint64, flow.Identifier] +} + +func NewHeaders(collector module.CacheMetrics, db *badger.DB) *Headers { + + store := func(blockID flow.Identifier, header *flow.Header) func(*transaction.Tx) error { + return transaction.WithTx(operation.InsertHeader(blockID, header)) + } + + // CAUTION: should only be used to index FINALIZED blocks by their + // respective height + storeHeight := func(height uint64, id flow.Identifier) func(*transaction.Tx) error { + return transaction.WithTx(operation.IndexBlockHeight(height, id)) + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.Header, error) { + var header flow.Header + return func(tx *badger.Txn) (*flow.Header, error) { + err := operation.RetrieveHeader(blockID, &header)(tx) + return &header, err + } + } + + retrieveHeight := func(height uint64) func(tx *badger.Txn) (flow.Identifier, error) { + return func(tx *badger.Txn) (flow.Identifier, error) { + var id flow.Identifier + err := operation.LookupBlockHeight(height, &id)(tx) + return id, err + } + } + + h := &Headers{ + db: db, + cache: newCache(collector, metrics.ResourceHeader, + withLimit[flow.Identifier, *flow.Header](4*flow.DefaultTransactionExpiry), + withStore(store), + withRetrieve(retrieve)), + + heightCache: newCache(collector, metrics.ResourceFinalizedHeight, + withLimit[uint64, flow.Identifier](4*flow.DefaultTransactionExpiry), + withStore(storeHeight), + withRetrieve(retrieveHeight)), + } + + return h +} + +func (h *Headers) storeTx(header *flow.Header) func(*transaction.Tx) error { + return h.cache.PutTx(header.ID(), header) +} + +func (h *Headers) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.Header, error) { + return func(tx *badger.Txn) (*flow.Header, error) { + val, err := h.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +// results in `storage.ErrNotFound` for unknown height +func (h *Headers) retrieveIdByHeightTx(height uint64) func(*badger.Txn) (flow.Identifier, error) { + return func(tx *badger.Txn) (flow.Identifier, error) { + blockID, err := h.heightCache.Get(height)(tx) + if err != nil { + return flow.ZeroID, fmt.Errorf("failed to retrieve block ID for height %d: %w", height, err) + } + return blockID, nil + } +} + +func (h *Headers) Store(header *flow.Header) error { + return operation.RetryOnConflictTx(h.db, transaction.Update, h.storeTx(header)) +} + +func (h *Headers) ByBlockID(blockID flow.Identifier) (*flow.Header, error) { + tx := h.db.NewTransaction(false) + defer tx.Discard() + return h.retrieveTx(blockID)(tx) +} + +func (h *Headers) ByHeight(height uint64) (*flow.Header, error) { + tx := h.db.NewTransaction(false) + defer tx.Discard() + + blockID, err := h.retrieveIdByHeightTx(height)(tx) + if err != nil { + return nil, err + } + return h.retrieveTx(blockID)(tx) +} + +// Exists returns true if a header with the given ID has been stored. +// No errors are expected during normal operation. +func (h *Headers) Exists(blockID flow.Identifier) (bool, error) { + // if the block is in the cache, return true + if ok := h.cache.IsCached(blockID); ok { + return ok, nil + } + // otherwise, check badger store + var exists bool + err := h.db.View(operation.BlockExists(blockID, &exists)) + if err != nil { + return false, fmt.Errorf("could not check existence: %w", err) + } + return exists, nil +} + +// BlockIDByHeight returns the block ID that is finalized at the given height. It is an optimized +// version of `ByHeight` that skips retrieving the block. Expected errors during normal operations: +// - `storage.ErrNotFound` if no finalized block is known at given height. +func (h *Headers) BlockIDByHeight(height uint64) (flow.Identifier, error) { + tx := h.db.NewTransaction(false) + defer tx.Discard() + + blockID, err := h.retrieveIdByHeightTx(height)(tx) + if err != nil { + return flow.ZeroID, fmt.Errorf("could not lookup block id by height %d: %w", height, err) + } + return blockID, nil +} + +func (h *Headers) ByParentID(parentID flow.Identifier) ([]*flow.Header, error) { + var blockIDs flow.IdentifierList + err := h.db.View(procedure.LookupBlockChildren(parentID, &blockIDs)) + if err != nil { + return nil, fmt.Errorf("could not look up children: %w", err) + } + headers := make([]*flow.Header, 0, len(blockIDs)) + for _, blockID := range blockIDs { + header, err := h.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("could not retrieve child (%x): %w", blockID, err) + } + headers = append(headers, header) + } + return headers, nil +} + +func (h *Headers) FindHeaders(filter func(header *flow.Header) bool) ([]flow.Header, error) { + blocks := make([]flow.Header, 0, 1) + err := h.db.View(operation.FindHeaders(filter, &blocks)) + return blocks, err +} + +// RollbackExecutedBlock update the executed block header to the given header. +// only useful for execution node to roll back executed block height +func (h *Headers) RollbackExecutedBlock(header *flow.Header) error { + return operation.RetryOnConflict(h.db.Update, func(txn *badger.Txn) error { + var blockID flow.Identifier + err := operation.RetrieveExecutedBlock(&blockID)(txn) + if err != nil { + return fmt.Errorf("cannot lookup executed block: %w", err) + } + + var highest flow.Header + err = operation.RetrieveHeader(blockID, &highest)(txn) + if err != nil { + return fmt.Errorf("cannot retrieve executed header: %w", err) + } + + // only rollback if the given height is below the current executed height + if header.Height >= highest.Height { + return fmt.Errorf("cannot roolback. expect the target height %v to be lower than highest executed height %v, but actually is not", + header.Height, highest.Height, + ) + } + + err = operation.UpdateExecutedBlock(header.ID())(txn) + if err != nil { + return fmt.Errorf("cannot update highest executed block: %w", err) + } + + return nil + }) +} diff --git a/storage/pebble/headers_test.go b/storage/pebble/headers_test.go new file mode 100644 index 00000000000..e0d55bec662 --- /dev/null +++ b/storage/pebble/headers_test.go @@ -0,0 +1,52 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/onflow/flow-go/storage/badger/operation" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestHeaderStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + headers := badgerstorage.NewHeaders(metrics, db) + + block := unittest.BlockFixture() + + // store header + err := headers.Store(block.Header) + require.NoError(t, err) + + // index the header + err = operation.RetryOnConflict(db.Update, operation.IndexBlockHeight(block.Header.Height, block.ID())) + require.NoError(t, err) + + // retrieve header by height + actual, err := headers.ByHeight(block.Header.Height) + require.NoError(t, err) + require.Equal(t, block.Header, actual) + }) +} + +func TestHeaderRetrieveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + headers := badgerstorage.NewHeaders(metrics, db) + + header := unittest.BlockHeaderFixture() + + // retrieve header by height, should err as not store before height + _, err := headers.ByHeight(header.Height) + require.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} diff --git a/storage/pebble/index.go b/storage/pebble/index.go new file mode 100644 index 00000000000..49d87b928da --- /dev/null +++ b/storage/pebble/index.go @@ -0,0 +1,69 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// Index implements a simple read-only payload storage around a badger DB. +type Index struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.Index] +} + +func NewIndex(collector module.CacheMetrics, db *badger.DB) *Index { + + store := func(blockID flow.Identifier, index *flow.Index) func(*transaction.Tx) error { + return transaction.WithTx(procedure.InsertIndex(blockID, index)) + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.Index, error) { + var index flow.Index + return func(tx *badger.Txn) (*flow.Index, error) { + err := procedure.RetrieveIndex(blockID, &index)(tx) + return &index, err + } + } + + p := &Index{ + db: db, + cache: newCache[flow.Identifier, *flow.Index](collector, metrics.ResourceIndex, + withLimit[flow.Identifier, *flow.Index](flow.DefaultTransactionExpiry+100), + withStore(store), + withRetrieve(retrieve)), + } + + return p +} + +func (i *Index) storeTx(blockID flow.Identifier, index *flow.Index) func(*transaction.Tx) error { + return i.cache.PutTx(blockID, index) +} + +func (i *Index) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.Index, error) { + return func(tx *badger.Txn) (*flow.Index, error) { + val, err := i.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +func (i *Index) Store(blockID flow.Identifier, index *flow.Index) error { + return operation.RetryOnConflictTx(i.db, transaction.Update, i.storeTx(blockID, index)) +} + +func (i *Index) ByBlockID(blockID flow.Identifier) (*flow.Index, error) { + tx := i.db.NewTransaction(false) + defer tx.Discard() + return i.retrieveTx(blockID)(tx) +} diff --git a/storage/pebble/index_test.go b/storage/pebble/index_test.go new file mode 100644 index 00000000000..ba4e2f3d6d8 --- /dev/null +++ b/storage/pebble/index_test.go @@ -0,0 +1,38 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestIndexStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewIndex(metrics, db) + + blockID := unittest.IdentifierFixture() + expected := unittest.IndexFixture() + + // retreive without store + _, err := store.ByBlockID(blockID) + require.True(t, errors.Is(err, storage.ErrNotFound)) + + // store index + err = store.Store(blockID, expected) + require.NoError(t, err) + + // retreive index + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/init.go b/storage/pebble/init.go new file mode 100644 index 00000000000..a3d4691bc83 --- /dev/null +++ b/storage/pebble/init.go @@ -0,0 +1,45 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/storage/badger/operation" +) + +// InitPublic initializes a public database by checking and setting the database +// type marker. If an existing, inconsistent type marker is set, this method will +// return an error. Once a database type marker has been set using these methods, +// the type cannot be changed. +func InitPublic(opts badger.Options) (*badger.DB, error) { + + db, err := badger.Open(opts) + if err != nil { + return nil, fmt.Errorf("could not open db: %w", err) + } + err = db.Update(operation.InsertPublicDBMarker) + if err != nil { + return nil, fmt.Errorf("could not assert db type: %w", err) + } + + return db, nil +} + +// InitSecret initializes a secrets database by checking and setting the database +// type marker. If an existing, inconsistent type marker is set, this method will +// return an error. Once a database type marker has been set using these methods, +// the type cannot be changed. +func InitSecret(opts badger.Options) (*badger.DB, error) { + + db, err := badger.Open(opts) + if err != nil { + return nil, fmt.Errorf("could not open db: %w", err) + } + err = db.Update(operation.InsertSecretDBMarker) + if err != nil { + return nil, fmt.Errorf("could not assert db type: %w", err) + } + + return db, nil +} diff --git a/storage/pebble/init_test.go b/storage/pebble/init_test.go new file mode 100644 index 00000000000..7392babce41 --- /dev/null +++ b/storage/pebble/init_test.go @@ -0,0 +1,56 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInitPublic(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitPublic, func(db *badger.DB) { + err := operation.EnsurePublicDB(db) + require.NoError(t, err) + err = operation.EnsureSecretDB(db) + require.Error(t, err) + }) +} + +func TestInitSecret(t *testing.T) { + unittest.RunWithTypedBadgerDB(t, bstorage.InitSecret, func(db *badger.DB) { + err := operation.EnsureSecretDB(db) + require.NoError(t, err) + err = operation.EnsurePublicDB(db) + require.Error(t, err) + }) +} + +// opening a database which has previously been opened with encryption enabled, +// using a different encryption key, should fail +func TestEncryptionKeyMismatch(t *testing.T) { + unittest.RunWithTempDir(t, func(dir string) { + + // open a database with encryption enabled + key1 := unittest.SeedFixture(32) + db := unittest.TypedBadgerDB(t, dir, func(options badger.Options) (*badger.DB, error) { + options = options.WithEncryptionKey(key1) + return badger.Open(options) + }) + db.Close() + + // open the same database with a different key + key2 := unittest.SeedFixture(32) + opts := badger. + DefaultOptions(dir). + WithKeepL0InMemory(true). + WithEncryptionKey(key2). + WithLogger(nil) + _, err := badger.Open(opts) + // opening the database should return an error + require.Error(t, err) + }) +} diff --git a/storage/pebble/light_transaction_results.go b/storage/pebble/light_transaction_results.go new file mode 100644 index 00000000000..13e8863a276 --- /dev/null +++ b/storage/pebble/light_transaction_results.go @@ -0,0 +1,160 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +var _ storage.LightTransactionResults = (*LightTransactionResults)(nil) + +type LightTransactionResults struct { + db *badger.DB + cache *Cache[string, flow.LightTransactionResult] + indexCache *Cache[string, flow.LightTransactionResult] + blockCache *Cache[string, []flow.LightTransactionResult] +} + +func NewLightTransactionResults(collector module.CacheMetrics, db *badger.DB, transactionResultsCacheSize uint) *LightTransactionResults { + retrieve := func(key string) func(tx *badger.Txn) (flow.LightTransactionResult, error) { + var txResult flow.LightTransactionResult + return func(tx *badger.Txn) (flow.LightTransactionResult, error) { + + blockID, txID, err := KeyToBlockIDTransactionID(key) + if err != nil { + return flow.LightTransactionResult{}, fmt.Errorf("could not convert key: %w", err) + } + + err = operation.RetrieveLightTransactionResult(blockID, txID, &txResult)(tx) + if err != nil { + return flow.LightTransactionResult{}, handleError(err, flow.LightTransactionResult{}) + } + return txResult, nil + } + } + retrieveIndex := func(key string) func(tx *badger.Txn) (flow.LightTransactionResult, error) { + var txResult flow.LightTransactionResult + return func(tx *badger.Txn) (flow.LightTransactionResult, error) { + + blockID, txIndex, err := KeyToBlockIDIndex(key) + if err != nil { + return flow.LightTransactionResult{}, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.RetrieveLightTransactionResultByIndex(blockID, txIndex, &txResult)(tx) + if err != nil { + return flow.LightTransactionResult{}, handleError(err, flow.LightTransactionResult{}) + } + return txResult, nil + } + } + retrieveForBlock := func(key string) func(tx *badger.Txn) ([]flow.LightTransactionResult, error) { + var txResults []flow.LightTransactionResult + return func(tx *badger.Txn) ([]flow.LightTransactionResult, error) { + + blockID, err := KeyToBlockID(key) + if err != nil { + return nil, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.LookupLightTransactionResultsByBlockIDUsingIndex(blockID, &txResults)(tx) + if err != nil { + return nil, handleError(err, flow.LightTransactionResult{}) + } + return txResults, nil + } + } + return &LightTransactionResults{ + db: db, + cache: newCache[string, flow.LightTransactionResult](collector, metrics.ResourceTransactionResults, + withLimit[string, flow.LightTransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.LightTransactionResult]), + withRetrieve(retrieve), + ), + indexCache: newCache[string, flow.LightTransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, flow.LightTransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.LightTransactionResult]), + withRetrieve(retrieveIndex), + ), + blockCache: newCache[string, []flow.LightTransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, []flow.LightTransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, []flow.LightTransactionResult]), + withRetrieve(retrieveForBlock), + ), + } +} + +func (tr *LightTransactionResults) BatchStore(blockID flow.Identifier, transactionResults []flow.LightTransactionResult, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + + for i, result := range transactionResults { + err := operation.BatchInsertLightTransactionResult(blockID, &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch insert tx result: %w", err) + } + + err = operation.BatchIndexLightTransactionResult(blockID, uint32(i), &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch index tx result: %w", err) + } + } + + batch.OnSucceed(func() { + for i, result := range transactionResults { + key := KeyFromBlockIDTransactionID(blockID, result.TransactionID) + // cache for each transaction, so that it's faster to retrieve + tr.cache.Insert(key, result) + + index := uint32(i) + + keyIndex := KeyFromBlockIDIndex(blockID, index) + tr.indexCache.Insert(keyIndex, result) + } + + key := KeyFromBlockID(blockID) + tr.blockCache.Insert(key, transactionResults) + }) + return nil +} + +// ByBlockIDTransactionID returns the transaction result for the given block ID and transaction ID +func (tr *LightTransactionResults) ByBlockIDTransactionID(blockID flow.Identifier, txID flow.Identifier) (*flow.LightTransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDTransactionID(blockID, txID) + transactionResult, err := tr.cache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResult, nil +} + +// ByBlockIDTransactionIndex returns the transaction result for the given blockID and transaction index +func (tr *LightTransactionResults) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.LightTransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDIndex(blockID, txIndex) + transactionResult, err := tr.indexCache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResult, nil +} + +// ByBlockID gets all transaction results for a block, ordered by transaction index +func (tr *LightTransactionResults) ByBlockID(blockID flow.Identifier) ([]flow.LightTransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockID(blockID) + transactionResults, err := tr.blockCache.Get(key)(tx) + if err != nil { + return nil, err + } + return transactionResults, nil +} diff --git a/storage/pebble/light_transaction_results_test.go b/storage/pebble/light_transaction_results_test.go new file mode 100644 index 00000000000..61fc857e0bb --- /dev/null +++ b/storage/pebble/light_transaction_results_test.go @@ -0,0 +1,115 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestBatchStoringLightTransactionResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewLightTransactionResults(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + txResults := getLightTransactionResultsFixture(10) + + t.Run("batch store results", func(t *testing.T) { + writeBatch := bstorage.NewBatch(db) + err := store.BatchStore(blockID, txResults, writeBatch) + require.NoError(t, err) + + err = writeBatch.Flush() + require.NoError(t, err) + + // add a results to a new block to validate they are not included in lookups + writeBatch = bstorage.NewBatch(db) + err = store.BatchStore(unittest.IdentifierFixture(), getLightTransactionResultsFixture(2), writeBatch) + require.NoError(t, err) + + err = writeBatch.Flush() + require.NoError(t, err) + }) + + t.Run("read results with cache", func(t *testing.T) { + for _, txResult := range txResults { + actual, err := store.ByBlockIDTransactionID(blockID, txResult.TransactionID) + require.NoError(t, err) + assert.Equal(t, txResult, *actual) + } + }) + + newStore := bstorage.NewLightTransactionResults(metrics, db, 1000) + t.Run("read results without cache", func(t *testing.T) { + // test loading from database (without cache) + // create a new instance using the same db so it has an empty cache + for _, txResult := range txResults { + actual, err := newStore.ByBlockIDTransactionID(blockID, txResult.TransactionID) + require.NoError(t, err) + assert.Equal(t, txResult, *actual) + } + }) + + t.Run("cached and non-cached results are equal", func(t *testing.T) { + // check retrieving by index from both cache and db + for i := len(txResults) - 1; i >= 0; i-- { + actual, err := store.ByBlockIDTransactionIndex(blockID, uint32(i)) + require.NoError(t, err) + assert.Equal(t, txResults[i], *actual) + + actual, err = newStore.ByBlockIDTransactionIndex(blockID, uint32(i)) + require.NoError(t, err) + assert.Equal(t, txResults[i], *actual) + } + }) + + t.Run("read all results for block", func(t *testing.T) { + actuals, err := store.ByBlockID(blockID) + require.NoError(t, err) + + assert.Equal(t, len(txResults), len(actuals)) + for i := range txResults { + assert.Equal(t, txResults[i], actuals[i]) + } + }) + }) +} + +func TestReadingNotStoredLightTransactionResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewLightTransactionResults(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + txID := unittest.IdentifierFixture() + txIndex := rand.Uint32() + + _, err := store.ByBlockIDTransactionID(blockID, txID) + assert.ErrorIs(t, err, storage.ErrNotFound) + + _, err = store.ByBlockIDTransactionIndex(blockID, txIndex) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) +} + +func getLightTransactionResultsFixture(n int) []flow.LightTransactionResult { + txResults := make([]flow.LightTransactionResult, 0, n) + for i := 0; i < n; i++ { + expected := flow.LightTransactionResult{ + TransactionID: unittest.IdentifierFixture(), + Failed: i%2 == 0, + ComputationUsed: unittest.Uint64InRange(1, 1000), + } + txResults = append(txResults, expected) + } + return txResults +} diff --git a/storage/pebble/my_receipts.go b/storage/pebble/my_receipts.go new file mode 100644 index 00000000000..ff1584f44d6 --- /dev/null +++ b/storage/pebble/my_receipts.go @@ -0,0 +1,159 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// MyExecutionReceipts holds and indexes Execution Receipts. +// MyExecutionReceipts is implemented as a wrapper around badger.ExecutionReceipts +// The wrapper adds the ability to "MY execution receipt", from the viewpoint +// of an individual Execution Node. +type MyExecutionReceipts struct { + genericReceipts *ExecutionReceipts + db *badger.DB + cache *Cache[flow.Identifier, *flow.ExecutionReceipt] +} + +// NewMyExecutionReceipts creates instance of MyExecutionReceipts which is a wrapper wrapper around badger.ExecutionReceipts +// It's useful for execution nodes to keep track of produced execution receipts. +func NewMyExecutionReceipts(collector module.CacheMetrics, db *badger.DB, receipts *ExecutionReceipts) *MyExecutionReceipts { + store := func(key flow.Identifier, receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { + // assemble DB operations to store receipt (no execution) + storeReceiptOps := receipts.storeTx(receipt) + // assemble DB operations to index receipt as one of my own (no execution) + blockID := receipt.ExecutionResult.BlockID + receiptID := receipt.ID() + indexOwnReceiptOps := transaction.WithTx(func(tx *badger.Txn) error { + err := operation.IndexOwnExecutionReceipt(blockID, receiptID)(tx) + // check if we are storing same receipt + if errors.Is(err, storage.ErrAlreadyExists) { + var savedReceiptID flow.Identifier + err := operation.LookupOwnExecutionReceipt(blockID, &savedReceiptID)(tx) + if err != nil { + return err + } + + if savedReceiptID == receiptID { + // if we are storing same receipt we shouldn't error + return nil + } + + return fmt.Errorf("indexing my receipt %v failed: different receipt %v for the same block %v is already indexed", receiptID, + savedReceiptID, blockID) + } + return err + }) + + return func(tx *transaction.Tx) error { + err := storeReceiptOps(tx) // execute operations to store receipt + if err != nil { + return fmt.Errorf("could not store receipt: %w", err) + } + err = indexOwnReceiptOps(tx) // execute operations to index receipt as one of my own + if err != nil { + return fmt.Errorf("could not index receipt as one of my own: %w", err) + } + return nil + } + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + var receiptID flow.Identifier + err := operation.LookupOwnExecutionReceipt(blockID, &receiptID)(tx) + if err != nil { + return nil, fmt.Errorf("could not lookup receipt ID: %w", err) + } + receipt, err := receipts.byID(receiptID)(tx) + if err != nil { + return nil, err + } + return receipt, nil + } + } + + return &MyExecutionReceipts{ + genericReceipts: receipts, + db: db, + cache: newCache[flow.Identifier, *flow.ExecutionReceipt](collector, metrics.ResourceMyReceipt, + withLimit[flow.Identifier, *flow.ExecutionReceipt](flow.DefaultTransactionExpiry+100), + withStore(store), + withRetrieve(retrieve)), + } +} + +// storeMyReceipt assembles the operations to store the receipt and marks it as mine (trusted). +func (m *MyExecutionReceipts) storeMyReceipt(receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { + return m.cache.PutTx(receipt.ExecutionResult.BlockID, receipt) +} + +// storeMyReceipt assembles the operations to retrieve my receipt for the given block ID. +func (m *MyExecutionReceipts) myReceipt(blockID flow.Identifier) func(*badger.Txn) (*flow.ExecutionReceipt, error) { + retrievalOps := m.cache.Get(blockID) // assemble DB operations to retrieve receipt (no execution) + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + val, err := retrievalOps(tx) // execute operations to retrieve receipt + if err != nil { + return nil, err + } + return val, nil + } +} + +// StoreMyReceipt stores the receipt and marks it as mine (trusted). My +// receipts are indexed by the block whose result they compute. Currently, +// we only support indexing a _single_ receipt per block. Attempting to +// store conflicting receipts for the same block will error. +func (m *MyExecutionReceipts) StoreMyReceipt(receipt *flow.ExecutionReceipt) error { + return operation.RetryOnConflictTx(m.db, transaction.Update, m.storeMyReceipt(receipt)) +} + +// BatchStoreMyReceipt stores blockID-to-my-receipt index entry keyed by blockID in a provided batch. +// No errors are expected during normal operation +// If entity fails marshalling, the error is wrapped in a generic error and returned. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (m *MyExecutionReceipts) BatchStoreMyReceipt(receipt *flow.ExecutionReceipt, batch storage.BatchStorage) error { + + writeBatch := batch.GetWriter() + + err := m.genericReceipts.BatchStore(receipt, batch) + if err != nil { + return fmt.Errorf("cannot batch store generic execution receipt inside my execution receipt batch store: %w", err) + } + + err = operation.BatchIndexOwnExecutionReceipt(receipt.ExecutionResult.BlockID, receipt.ID())(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch index own execution receipt inside my execution receipt batch store: %w", err) + } + + return nil +} + +// MyReceipt retrieves my receipt for the given block. +// Returns storage.ErrNotFound if no receipt was persisted for the block. +func (m *MyExecutionReceipts) MyReceipt(blockID flow.Identifier) (*flow.ExecutionReceipt, error) { + tx := m.db.NewTransaction(false) + defer tx.Discard() + return m.myReceipt(blockID)(tx) +} + +func (m *MyExecutionReceipts) RemoveIndexByBlockID(blockID flow.Identifier) error { + return m.db.Update(operation.SkipNonExist(operation.RemoveOwnExecutionReceipt(blockID))) +} + +// BatchRemoveIndexByBlockID removes blockID-to-my-execution-receipt index entry keyed by a blockID in a provided batch +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (m *MyExecutionReceipts) BatchRemoveIndexByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return operation.BatchRemoveOwnExecutionReceipt(blockID)(writeBatch) +} diff --git a/storage/pebble/my_receipts_test.go b/storage/pebble/my_receipts_test.go new file mode 100644 index 00000000000..942c771f041 --- /dev/null +++ b/storage/pebble/my_receipts_test.go @@ -0,0 +1,72 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestMyExecutionReceiptsStorage(t *testing.T) { + withStore := func(t *testing.T, f func(store *bstorage.MyExecutionReceipts)) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + results := bstorage.NewExecutionResults(metrics, db) + receipts := bstorage.NewExecutionReceipts(metrics, db, results, bstorage.DefaultCacheSize) + store := bstorage.NewMyExecutionReceipts(metrics, db, receipts) + + f(store) + }) + } + + t.Run("store one get one", func(t *testing.T) { + withStore(t, func(store *bstorage.MyExecutionReceipts) { + block := unittest.BlockFixture() + receipt1 := unittest.ReceiptForBlockFixture(&block) + + err := store.StoreMyReceipt(receipt1) + require.NoError(t, err) + + actual, err := store.MyReceipt(block.ID()) + require.NoError(t, err) + + require.Equal(t, receipt1, actual) + }) + }) + + t.Run("store same for the same block", func(t *testing.T) { + withStore(t, func(store *bstorage.MyExecutionReceipts) { + block := unittest.BlockFixture() + + receipt1 := unittest.ReceiptForBlockFixture(&block) + + err := store.StoreMyReceipt(receipt1) + require.NoError(t, err) + + err = store.StoreMyReceipt(receipt1) + require.NoError(t, err) + }) + }) + + t.Run("store different receipt for same block should fail", func(t *testing.T) { + withStore(t, func(store *bstorage.MyExecutionReceipts) { + block := unittest.BlockFixture() + + executor1 := unittest.IdentifierFixture() + executor2 := unittest.IdentifierFixture() + + receipt1 := unittest.ReceiptForBlockExecutorFixture(&block, executor1) + receipt2 := unittest.ReceiptForBlockExecutorFixture(&block, executor2) + + err := store.StoreMyReceipt(receipt1) + require.NoError(t, err) + + err = store.StoreMyReceipt(receipt2) + require.Error(t, err) + }) + }) +} diff --git a/storage/pebble/operation/approvals.go b/storage/pebble/operation/approvals.go new file mode 100644 index 00000000000..8a994eed2a2 --- /dev/null +++ b/storage/pebble/operation/approvals.go @@ -0,0 +1,31 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertResultApproval inserts a ResultApproval by ID. +func InsertResultApproval(approval *flow.ResultApproval) func(*badger.Txn) error { + return insert(makePrefix(codeResultApproval, approval.ID()), approval) +} + +// RetrieveResultApproval retrieves an approval by ID. +func RetrieveResultApproval(approvalID flow.Identifier, approval *flow.ResultApproval) func(*badger.Txn) error { + return retrieve(makePrefix(codeResultApproval, approvalID), approval) +} + +// IndexResultApproval inserts a ResultApproval ID keyed by ExecutionResult ID +// and chunk index. If a value for this key exists, a storage.ErrAlreadyExists +// error is returned. This operation is only used by the ResultApprovals store, +// which is only used within a Verification node, where it is assumed that there +// is only one approval per chunk. +func IndexResultApproval(resultID flow.Identifier, chunkIndex uint64, approvalID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeIndexResultApprovalByChunk, resultID, chunkIndex), approvalID) +} + +// LookupResultApproval finds a ResultApproval by result ID and chunk index. +func LookupResultApproval(resultID flow.Identifier, chunkIndex uint64, approvalID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeIndexResultApprovalByChunk, resultID, chunkIndex), approvalID) +} diff --git a/storage/pebble/operation/bft.go b/storage/pebble/operation/bft.go new file mode 100644 index 00000000000..8a6c8d2e8b3 --- /dev/null +++ b/storage/pebble/operation/bft.go @@ -0,0 +1,42 @@ +package operation + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +// PurgeBlocklist removes the set of blocked nodes IDs from the data base. +// If no corresponding entry exists, this function is a no-op. +// No errors are expected during normal operations. +// TODO: TEMPORARY manual override for adding node IDs to list of ejected nodes, applies to networking layer only +func PurgeBlocklist() func(*badger.Txn) error { + return func(tx *badger.Txn) error { + err := remove(makePrefix(blockedNodeIDs))(tx) + if err != nil && !errors.Is(err, storage.ErrNotFound) { + return fmt.Errorf("enexpected error while purging blocklist: %w", err) + } + return nil + } +} + +// PersistBlocklist writes the set of blocked nodes IDs into the data base. +// If an entry already exists, it is overwritten; otherwise a new entry is created. +// No errors are expected during normal operations. +// +// TODO: TEMPORARY manual override for adding node IDs to list of ejected nodes, applies to networking layer only +func PersistBlocklist(blocklist map[flow.Identifier]struct{}) func(*badger.Txn) error { + return upsert(makePrefix(blockedNodeIDs), blocklist) +} + +// RetrieveBlocklist reads the set of blocked node IDs from the data base. +// Returns `storage.ErrNotFound` error in case no respective data base entry is present. +// +// TODO: TEMPORARY manual override for adding node IDs to list of ejected nodes, applies to networking layer only +func RetrieveBlocklist(blocklist *map[flow.Identifier]struct{}) func(*badger.Txn) error { + return retrieve(makePrefix(blockedNodeIDs), blocklist) +} diff --git a/storage/pebble/operation/bft_test.go b/storage/pebble/operation/bft_test.go new file mode 100644 index 00000000000..f1b573659fc --- /dev/null +++ b/storage/pebble/operation/bft_test.go @@ -0,0 +1,95 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +// Test_PersistBlocklist tests the operations: +// - PersistBlocklist(blocklist map[flow.Identifier]struct{}) +// - RetrieveBlocklist(blocklist *map[flow.Identifier]struct{}) +// - PurgeBlocklist() +func Test_PersistBlocklist(t *testing.T) { + t.Run("Retrieving non-existing blocklist should return 'storage.ErrNotFound'", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var blocklist map[flow.Identifier]struct{} + err := db.View(RetrieveBlocklist(&blocklist)) + require.ErrorIs(t, err, storage.ErrNotFound) + + }) + }) + + t.Run("Persisting and read blocklist", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blocklist := unittest.IdentifierListFixture(8).Lookup() + err := db.Update(PersistBlocklist(blocklist)) + require.NoError(t, err) + + var b map[flow.Identifier]struct{} + err = db.View(RetrieveBlocklist(&b)) + require.NoError(t, err) + require.Equal(t, blocklist, b) + }) + }) + + t.Run("Overwrite blocklist", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blocklist1 := unittest.IdentifierListFixture(8).Lookup() + err := db.Update(PersistBlocklist(blocklist1)) + require.NoError(t, err) + + blocklist2 := unittest.IdentifierListFixture(8).Lookup() + err = db.Update(PersistBlocklist(blocklist2)) + require.NoError(t, err) + + var b map[flow.Identifier]struct{} + err = db.View(RetrieveBlocklist(&b)) + require.NoError(t, err) + require.Equal(t, blocklist2, b) + }) + }) + + t.Run("Write & Purge & Write blocklist", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blocklist1 := unittest.IdentifierListFixture(8).Lookup() + err := db.Update(PersistBlocklist(blocklist1)) + require.NoError(t, err) + + err = db.Update(PurgeBlocklist()) + require.NoError(t, err) + + var b map[flow.Identifier]struct{} + err = db.View(RetrieveBlocklist(&b)) + require.ErrorIs(t, err, storage.ErrNotFound) + + blocklist2 := unittest.IdentifierListFixture(8).Lookup() + err = db.Update(PersistBlocklist(blocklist2)) + require.NoError(t, err) + + err = db.View(RetrieveBlocklist(&b)) + require.NoError(t, err) + require.Equal(t, blocklist2, b) + }) + }) + + t.Run("Purge non-existing blocklist", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var b map[flow.Identifier]struct{} + + err := db.View(RetrieveBlocklist(&b)) + require.ErrorIs(t, err, storage.ErrNotFound) + + err = db.Update(PurgeBlocklist()) + require.NoError(t, err) + + err = db.View(RetrieveBlocklist(&b)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) + }) +} diff --git a/storage/pebble/operation/children.go b/storage/pebble/operation/children.go new file mode 100644 index 00000000000..92eb0c35918 --- /dev/null +++ b/storage/pebble/operation/children.go @@ -0,0 +1,22 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertBlockChildren insert an index to lookup the direct child of a block by its ID +func InsertBlockChildren(blockID flow.Identifier, childrenIDs flow.IdentifierList) func(*badger.Txn) error { + return insert(makePrefix(codeBlockChildren, blockID), childrenIDs) +} + +// UpdateBlockChildren updates the children for a block. +func UpdateBlockChildren(blockID flow.Identifier, childrenIDs flow.IdentifierList) func(*badger.Txn) error { + return update(makePrefix(codeBlockChildren, blockID), childrenIDs) +} + +// RetrieveBlockChildren the child block ID by parent block ID +func RetrieveBlockChildren(blockID flow.Identifier, childrenIDs *flow.IdentifierList) func(*badger.Txn) error { + return retrieve(makePrefix(codeBlockChildren, blockID), childrenIDs) +} diff --git a/storage/pebble/operation/children_test.go b/storage/pebble/operation/children_test.go new file mode 100644 index 00000000000..629488373aa --- /dev/null +++ b/storage/pebble/operation/children_test.go @@ -0,0 +1,33 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestBlockChildrenIndexUpdateLookup(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blockID := unittest.IdentifierFixture() + childrenIDs := unittest.IdentifierListFixture(8) + var retrievedIDs flow.IdentifierList + + err := db.Update(InsertBlockChildren(blockID, childrenIDs)) + require.NoError(t, err) + err = db.View(RetrieveBlockChildren(blockID, &retrievedIDs)) + require.NoError(t, err) + assert.Equal(t, childrenIDs, retrievedIDs) + + altIDs := unittest.IdentifierListFixture(4) + err = db.Update(UpdateBlockChildren(blockID, altIDs)) + require.NoError(t, err) + err = db.View(RetrieveBlockChildren(blockID, &retrievedIDs)) + require.NoError(t, err) + assert.Equal(t, altIDs, retrievedIDs) + }) +} diff --git a/storage/pebble/operation/chunkDataPacks.go b/storage/pebble/operation/chunkDataPacks.go new file mode 100644 index 00000000000..e0f2deb2ce2 --- /dev/null +++ b/storage/pebble/operation/chunkDataPacks.go @@ -0,0 +1,35 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +// InsertChunkDataPack inserts a chunk data pack keyed by chunk ID. +func InsertChunkDataPack(c *storage.StoredChunkDataPack) func(*badger.Txn) error { + return insert(makePrefix(codeChunkDataPack, c.ChunkID), c) +} + +// BatchInsertChunkDataPack inserts a chunk data pack keyed by chunk ID into a batch +func BatchInsertChunkDataPack(c *storage.StoredChunkDataPack) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeChunkDataPack, c.ChunkID), c) +} + +// BatchRemoveChunkDataPack removes a chunk data pack keyed by chunk ID, in a batch. +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func BatchRemoveChunkDataPack(chunkID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchRemove(makePrefix(codeChunkDataPack, chunkID)) +} + +// RetrieveChunkDataPack retrieves a chunk data pack by chunk ID. +func RetrieveChunkDataPack(chunkID flow.Identifier, c *storage.StoredChunkDataPack) func(*badger.Txn) error { + return retrieve(makePrefix(codeChunkDataPack, chunkID), c) +} + +// RemoveChunkDataPack removes the chunk data pack with the given chunk ID. +func RemoveChunkDataPack(chunkID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeChunkDataPack, chunkID)) +} diff --git a/storage/pebble/operation/chunkDataPacks_test.go b/storage/pebble/operation/chunkDataPacks_test.go new file mode 100644 index 00000000000..f3a90af8d00 --- /dev/null +++ b/storage/pebble/operation/chunkDataPacks_test.go @@ -0,0 +1,50 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestChunkDataPack(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + collectionID := unittest.IdentifierFixture() + expected := &storage.StoredChunkDataPack{ + ChunkID: unittest.IdentifierFixture(), + StartState: unittest.StateCommitmentFixture(), + Proof: []byte{'p'}, + CollectionID: collectionID, + } + + t.Run("Retrieve non-existent", func(t *testing.T) { + var actual storage.StoredChunkDataPack + err := db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) + assert.Error(t, err) + }) + + t.Run("Save", func(t *testing.T) { + err := db.Update(InsertChunkDataPack(expected)) + require.NoError(t, err) + + var actual storage.StoredChunkDataPack + err = db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) + assert.NoError(t, err) + + assert.Equal(t, *expected, actual) + }) + + t.Run("Remove", func(t *testing.T) { + err := db.Update(RemoveChunkDataPack(expected.ChunkID)) + require.NoError(t, err) + + var actual storage.StoredChunkDataPack + err = db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) + assert.Error(t, err) + }) + }) +} diff --git a/storage/pebble/operation/chunk_locators.go b/storage/pebble/operation/chunk_locators.go new file mode 100644 index 00000000000..ef7f11fec50 --- /dev/null +++ b/storage/pebble/operation/chunk_locators.go @@ -0,0 +1,16 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/chunks" + "github.com/onflow/flow-go/model/flow" +) + +func InsertChunkLocator(locator *chunks.Locator) func(*badger.Txn) error { + return insert(makePrefix(codeChunk, locator.ID()), locator) +} + +func RetrieveChunkLocator(locatorID flow.Identifier, locator *chunks.Locator) func(*badger.Txn) error { + return retrieve(makePrefix(codeChunk, locatorID), locator) +} diff --git a/storage/pebble/operation/cluster.go b/storage/pebble/operation/cluster.go new file mode 100644 index 00000000000..8163285c62f --- /dev/null +++ b/storage/pebble/operation/cluster.go @@ -0,0 +1,83 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// This file implements storage functions for chain state book-keeping of +// collection node cluster consensus. In contrast to the corresponding functions +// for regular consensus, these functions include the cluster ID in order to +// support storing multiple chains, for example during epoch switchover. + +// IndexClusterBlockHeight inserts a block number to block ID mapping for +// the given cluster. +func IndexClusterBlockHeight(clusterID flow.ChainID, number uint64, blockID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeFinalizedCluster, clusterID, number), blockID) +} + +// LookupClusterBlockHeight retrieves a block ID by number for the given cluster +func LookupClusterBlockHeight(clusterID flow.ChainID, number uint64, blockID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeFinalizedCluster, clusterID, number), blockID) +} + +// InsertClusterFinalizedHeight inserts the finalized boundary for the given cluster. +func InsertClusterFinalizedHeight(clusterID flow.ChainID, number uint64) func(*badger.Txn) error { + return insert(makePrefix(codeClusterHeight, clusterID), number) +} + +// UpdateClusterFinalizedHeight updates the finalized boundary for the given cluster. +func UpdateClusterFinalizedHeight(clusterID flow.ChainID, number uint64) func(*badger.Txn) error { + return update(makePrefix(codeClusterHeight, clusterID), number) +} + +// RetrieveClusterFinalizedHeight retrieves the finalized boundary for the given cluster. +func RetrieveClusterFinalizedHeight(clusterID flow.ChainID, number *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeClusterHeight, clusterID), number) +} + +// IndexReferenceBlockByClusterBlock inserts the reference block ID for the given +// cluster block ID. While each cluster block specifies a reference block in its +// payload, we maintain this additional lookup for performance reasons. +func IndexReferenceBlockByClusterBlock(clusterBlockID, refID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeClusterBlockToRefBlock, clusterBlockID), refID) +} + +// LookupReferenceBlockByClusterBlock looks up the reference block ID for the given +// cluster block ID. While each cluster block specifies a reference block in its +// payload, we maintain this additional lookup for performance reasons. +func LookupReferenceBlockByClusterBlock(clusterBlockID flow.Identifier, refID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeClusterBlockToRefBlock, clusterBlockID), refID) +} + +// IndexClusterBlockByReferenceHeight indexes a cluster block ID by its reference +// block height. The cluster block ID is included in the key for more efficient +// traversal. Only finalized cluster blocks should be included in this index. +// The key looks like: +func IndexClusterBlockByReferenceHeight(refHeight uint64, clusterBlockID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeRefHeightToClusterBlock, refHeight, clusterBlockID), nil) +} + +// LookupClusterBlocksByReferenceHeightRange traverses the ref_height->cluster_block +// index and returns any finalized cluster blocks which have a reference block with +// height in the given range. This is used to avoid including duplicate transaction +// when building or validating a new collection. +func LookupClusterBlocksByReferenceHeightRange(start, end uint64, clusterBlockIDs *[]flow.Identifier) func(*badger.Txn) error { + startPrefix := makePrefix(codeRefHeightToClusterBlock, start) + endPrefix := makePrefix(codeRefHeightToClusterBlock, end) + prefixLen := len(startPrefix) + + return iterate(startPrefix, endPrefix, func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + clusterBlockIDBytes := key[prefixLen:] + var clusterBlockID flow.Identifier + copy(clusterBlockID[:], clusterBlockIDBytes) + *clusterBlockIDs = append(*clusterBlockIDs, clusterBlockID) + + // the info we need is stored in the key, never process the value + return false + } + return check, nil, nil + }, withPrefetchValuesFalse) +} diff --git a/storage/pebble/operation/cluster_test.go b/storage/pebble/operation/cluster_test.go new file mode 100644 index 00000000000..9a616e08490 --- /dev/null +++ b/storage/pebble/operation/cluster_test.go @@ -0,0 +1,313 @@ +package operation_test + +import ( + "errors" + "fmt" + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestClusterHeights(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var ( + clusterID flow.ChainID = "cluster" + height uint64 = 42 + expected = unittest.IdentifierFixture() + err error + ) + + t.Run("retrieve non-existent", func(t *testing.T) { + var actual flow.Identifier + err = db.View(operation.LookupClusterBlockHeight(clusterID, height, &actual)) + t.Log(err) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) + + t.Run("insert/retrieve", func(t *testing.T) { + err = db.Update(operation.IndexClusterBlockHeight(clusterID, height, expected)) + assert.Nil(t, err) + + var actual flow.Identifier + err = db.View(operation.LookupClusterBlockHeight(clusterID, height, &actual)) + assert.Nil(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("multiple chain IDs", func(t *testing.T) { + for i := 0; i < 3; i++ { + // use different cluster ID but same block height + clusterID = flow.ChainID(fmt.Sprintf("cluster-%d", i)) + expected = unittest.IdentifierFixture() + + var actual flow.Identifier + err = db.View(operation.LookupClusterBlockHeight(clusterID, height, &actual)) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + err = db.Update(operation.IndexClusterBlockHeight(clusterID, height, expected)) + assert.Nil(t, err) + + err = db.View(operation.LookupClusterBlockHeight(clusterID, height, &actual)) + assert.Nil(t, err) + assert.Equal(t, expected, actual) + } + }) + }) +} + +func TestClusterBoundaries(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var ( + clusterID flow.ChainID = "cluster" + expected uint64 = 42 + err error + ) + + t.Run("retrieve non-existant", func(t *testing.T) { + var actual uint64 + err = db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &actual)) + t.Log(err) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) + + t.Run("insert/retrieve", func(t *testing.T) { + err = db.Update(operation.InsertClusterFinalizedHeight(clusterID, 21)) + assert.Nil(t, err) + + err = db.Update(operation.UpdateClusterFinalizedHeight(clusterID, expected)) + assert.Nil(t, err) + + var actual uint64 + err = db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &actual)) + assert.Nil(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("multiple chain IDs", func(t *testing.T) { + for i := 0; i < 3; i++ { + // use different cluster ID but same boundary + clusterID = flow.ChainID(fmt.Sprintf("cluster-%d", i)) + expected = uint64(i) + + var actual uint64 + err = db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &actual)) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + + err = db.Update(operation.InsertClusterFinalizedHeight(clusterID, expected)) + assert.Nil(t, err) + + err = db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &actual)) + assert.Nil(t, err) + assert.Equal(t, expected, actual) + } + }) + }) +} + +func TestClusterBlockByReferenceHeight(t *testing.T) { + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("should be able to index cluster block by reference height", func(t *testing.T) { + id := unittest.IdentifierFixture() + height := rand.Uint64() + err := db.Update(operation.IndexClusterBlockByReferenceHeight(height, id)) + assert.NoError(t, err) + + var retrieved []flow.Identifier + err = db.View(operation.LookupClusterBlocksByReferenceHeightRange(height, height, &retrieved)) + assert.NoError(t, err) + require.Len(t, retrieved, 1) + assert.Equal(t, id, retrieved[0]) + }) + }) + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("should be able to index multiple cluster blocks at same reference height", func(t *testing.T) { + ids := unittest.IdentifierListFixture(10) + height := rand.Uint64() + for _, id := range ids { + err := db.Update(operation.IndexClusterBlockByReferenceHeight(height, id)) + assert.NoError(t, err) + } + + var retrieved []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(height, height, &retrieved)) + assert.NoError(t, err) + assert.Len(t, retrieved, len(ids)) + assert.ElementsMatch(t, ids, retrieved) + }) + }) + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("should be able to lookup cluster blocks across height range", func(t *testing.T) { + ids := unittest.IdentifierListFixture(100) + nextHeight := rand.Uint64() + // keep track of height range + minHeight, maxHeight := nextHeight, nextHeight + // keep track of which ids are indexed at each nextHeight + lookup := make(map[uint64][]flow.Identifier) + + for i := 0; i < len(ids); i++ { + // randomly adjust the nextHeight, increasing on average + r := rand.Intn(100) + if r < 20 { + nextHeight -= 1 // 20% + } else if r < 40 { + // nextHeight stays the same - 20% + } else if r < 80 { + nextHeight += 1 // 40% + } else { + nextHeight += 2 // 20% + } + + lookup[nextHeight] = append(lookup[nextHeight], ids[i]) + if nextHeight < minHeight { + minHeight = nextHeight + } + if nextHeight > maxHeight { + maxHeight = nextHeight + } + + err := db.Update(operation.IndexClusterBlockByReferenceHeight(nextHeight, ids[i])) + assert.NoError(t, err) + } + + // determine which ids we expect to be retrieved for a given height range + idsInHeightRange := func(min, max uint64) []flow.Identifier { + var idsForHeight []flow.Identifier + for height, id := range lookup { + if min <= height && height <= max { + idsForHeight = append(idsForHeight, id...) + } + } + return idsForHeight + } + + // Test cases are described as follows: + // {---} represents the queried height range + // [---] represents the indexed height range + // [{ means the left endpoint of both ranges are the same + // {-[ means the left endpoint of the queried range is strictly less than the indexed range + t.Run("{-}--[-]", func(t *testing.T) { + var retrieved []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(minHeight-100, minHeight-1, &retrieved)) + assert.NoError(t, err) + assert.Len(t, retrieved, 0) + }) + t.Run("{-[--}-]", func(t *testing.T) { + var retrieved []flow.Identifier + min := minHeight - 100 + max := minHeight + (maxHeight-minHeight)/2 + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(min, max, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(min, max) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("{[--}--]", func(t *testing.T) { + var retrieved []flow.Identifier + min := minHeight + max := minHeight + (maxHeight-minHeight)/2 + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(min, max, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(min, max) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("[-{--}-]", func(t *testing.T) { + var retrieved []flow.Identifier + min := minHeight + 1 + max := maxHeight - 1 + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(min, max, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(min, max) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("[{----}]", func(t *testing.T) { + var retrieved []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(minHeight, maxHeight, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(minHeight, maxHeight) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("[--{--}]", func(t *testing.T) { + var retrieved []flow.Identifier + min := minHeight + (maxHeight-minHeight)/2 + max := maxHeight + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(min, max, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(min, max) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("[-{--]-}", func(t *testing.T) { + var retrieved []flow.Identifier + min := minHeight + (maxHeight-minHeight)/2 + max := maxHeight + 100 + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(min, max, &retrieved)) + assert.NoError(t, err) + + expected := idsInHeightRange(min, max) + assert.NotEmpty(t, expected, "test assumption broken") + assert.Len(t, retrieved, len(expected)) + assert.ElementsMatch(t, expected, retrieved) + }) + t.Run("[-]--{-}", func(t *testing.T) { + var retrieved []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(maxHeight+1, maxHeight+100, &retrieved)) + assert.NoError(t, err) + assert.Len(t, retrieved, 0) + }) + }) + }) +} + +// expected average case # of blocks to lookup on Mainnet +func BenchmarkLookupClusterBlocksByReferenceHeightRange_1200(b *testing.B) { + benchmarkLookupClusterBlocksByReferenceHeightRange(b, 1200) +} + +// 5x average case on Mainnet +func BenchmarkLookupClusterBlocksByReferenceHeightRange_6_000(b *testing.B) { + benchmarkLookupClusterBlocksByReferenceHeightRange(b, 6_000) +} + +func BenchmarkLookupClusterBlocksByReferenceHeightRange_100_000(b *testing.B) { + benchmarkLookupClusterBlocksByReferenceHeightRange(b, 100_000) +} + +func benchmarkLookupClusterBlocksByReferenceHeightRange(b *testing.B, n int) { + unittest.RunWithBadgerDB(b, func(db *badger.DB) { + for i := 0; i < n; i++ { + err := db.Update(operation.IndexClusterBlockByReferenceHeight(rand.Uint64()%1000, unittest.IdentifierFixture())) + require.NoError(b, err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + var blockIDs []flow.Identifier + err := db.View(operation.LookupClusterBlocksByReferenceHeightRange(0, 1000, &blockIDs)) + require.NoError(b, err) + } + }) +} diff --git a/storage/pebble/operation/collections.go b/storage/pebble/operation/collections.go new file mode 100644 index 00000000000..4b8e0faf761 --- /dev/null +++ b/storage/pebble/operation/collections.go @@ -0,0 +1,46 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// NOTE: These insert light collections, which only contain references +// to the constituent transactions. They do not modify transactions contained +// by the collections. + +func InsertCollection(collection *flow.LightCollection) func(*badger.Txn) error { + return insert(makePrefix(codeCollection, collection.ID()), collection) +} + +func RetrieveCollection(collID flow.Identifier, collection *flow.LightCollection) func(*badger.Txn) error { + return retrieve(makePrefix(codeCollection, collID), collection) +} + +func RemoveCollection(collID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeCollection, collID)) +} + +// IndexCollectionPayload indexes the transactions within the collection payload +// of a cluster block. +func IndexCollectionPayload(blockID flow.Identifier, txIDs []flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeIndexCollection, blockID), txIDs) +} + +// LookupCollection looks up the collection for a given cluster payload. +func LookupCollectionPayload(blockID flow.Identifier, txIDs *[]flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeIndexCollection, blockID), txIDs) +} + +// IndexCollectionByTransaction inserts a collection id keyed by a transaction id +func IndexCollectionByTransaction(txID flow.Identifier, collectionID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeIndexCollectionByTransaction, txID), collectionID) +} + +// LookupCollectionID retrieves a collection id by transaction id +func RetrieveCollectionID(txID flow.Identifier, collectionID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeIndexCollectionByTransaction, txID), collectionID) +} diff --git a/storage/pebble/operation/collections_test.go b/storage/pebble/operation/collections_test.go new file mode 100644 index 00000000000..9bbe14386c8 --- /dev/null +++ b/storage/pebble/operation/collections_test.go @@ -0,0 +1,80 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestCollections(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.CollectionFixture(2).Light() + + t.Run("Retrieve nonexistant", func(t *testing.T) { + var actual flow.LightCollection + err := db.View(RetrieveCollection(expected.ID(), &actual)) + assert.Error(t, err) + }) + + t.Run("Save", func(t *testing.T) { + err := db.Update(InsertCollection(&expected)) + require.NoError(t, err) + + var actual flow.LightCollection + err = db.View(RetrieveCollection(expected.ID(), &actual)) + assert.NoError(t, err) + + assert.Equal(t, expected, actual) + }) + + t.Run("Remove", func(t *testing.T) { + err := db.Update(RemoveCollection(expected.ID())) + require.NoError(t, err) + + var actual flow.LightCollection + err = db.View(RetrieveCollection(expected.ID(), &actual)) + assert.Error(t, err) + }) + + t.Run("Index and lookup", func(t *testing.T) { + expected := unittest.CollectionFixture(1).Light() + blockID := unittest.IdentifierFixture() + + _ = db.Update(func(tx *badger.Txn) error { + err := InsertCollection(&expected)(tx) + assert.Nil(t, err) + err = IndexCollectionPayload(blockID, expected.Transactions)(tx) + assert.Nil(t, err) + return nil + }) + + var actual flow.LightCollection + err := db.View(LookupCollectionPayload(blockID, &actual.Transactions)) + assert.Nil(t, err) + + assert.Equal(t, expected, actual) + }) + + t.Run("Index and lookup by transaction ID", func(t *testing.T) { + expected := unittest.IdentifierFixture() + transactionID := unittest.IdentifierFixture() + actual := flow.Identifier{} + + _ = db.Update(func(tx *badger.Txn) error { + err := IndexCollectionByTransaction(transactionID, expected)(tx) + assert.Nil(t, err) + err = RetrieveCollectionID(transactionID, &actual)(tx) + assert.Nil(t, err) + return nil + }) + assert.Equal(t, expected, actual) + }) + }) +} diff --git a/storage/pebble/operation/commits.go b/storage/pebble/operation/commits.go new file mode 100644 index 00000000000..c7f13afd49f --- /dev/null +++ b/storage/pebble/operation/commits.go @@ -0,0 +1,42 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// IndexStateCommitment indexes a state commitment. +// +// State commitments are keyed by the block whose execution results in the state with the given commit. +func IndexStateCommitment(blockID flow.Identifier, commit flow.StateCommitment) func(*badger.Txn) error { + return insert(makePrefix(codeCommit, blockID), commit) +} + +// BatchIndexStateCommitment indexes a state commitment into a batch +// +// State commitments are keyed by the block whose execution results in the state with the given commit. +func BatchIndexStateCommitment(blockID flow.Identifier, commit flow.StateCommitment) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeCommit, blockID), commit) +} + +// LookupStateCommitment gets a state commitment keyed by block ID +// +// State commitments are keyed by the block whose execution results in the state with the given commit. +func LookupStateCommitment(blockID flow.Identifier, commit *flow.StateCommitment) func(*badger.Txn) error { + return retrieve(makePrefix(codeCommit, blockID), commit) +} + +// RemoveStateCommitment removes the state commitment by block ID +func RemoveStateCommitment(blockID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeCommit, blockID)) +} + +// BatchRemoveStateCommitment batch removes the state commitment by block ID +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func BatchRemoveStateCommitment(blockID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchRemove(makePrefix(codeCommit, blockID)) +} diff --git a/storage/pebble/operation/commits_test.go b/storage/pebble/operation/commits_test.go new file mode 100644 index 00000000000..392331e935a --- /dev/null +++ b/storage/pebble/operation/commits_test.go @@ -0,0 +1,26 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestStateCommitments(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.StateCommitmentFixture() + id := unittest.IdentifierFixture() + err := db.Update(IndexStateCommitment(id, expected)) + require.Nil(t, err) + + var actual flow.StateCommitment + err = db.View(LookupStateCommitment(id, &actual)) + require.Nil(t, err) + assert.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/operation/common_test.go b/storage/pebble/operation/common_test.go new file mode 100644 index 00000000000..65f64fbd5cb --- /dev/null +++ b/storage/pebble/operation/common_test.go @@ -0,0 +1,704 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "bytes" + "fmt" + "reflect" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/msgpack/v4" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +type Entity struct { + ID uint64 +} + +type UnencodeableEntity Entity + +var errCantEncode = fmt.Errorf("encoding not supported") +var errCantDecode = fmt.Errorf("decoding not supported") + +func (a UnencodeableEntity) MarshalJSON() ([]byte, error) { + return nil, errCantEncode +} + +func (a *UnencodeableEntity) UnmarshalJSON(b []byte) error { + return errCantDecode +} + +func (a UnencodeableEntity) MarshalMsgpack() ([]byte, error) { + return nil, errCantEncode +} + +func (a UnencodeableEntity) UnmarshalMsgpack(b []byte) error { + return errCantDecode +} + +func TestInsertValid(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + err := db.Update(insert(key, e)) + require.NoError(t, err) + + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + }) +} + +func TestInsertDuplicate(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + // persist first time + err := db.Update(insert(key, e)) + require.NoError(t, err) + + e2 := Entity{ID: 1338} + + // persist again + err = db.Update(insert(key, e2)) + require.Error(t, err) + require.ErrorIs(t, err, storage.ErrAlreadyExists) + + // ensure old value did not update + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + }) +} + +func TestInsertEncodingError(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + + err := db.Update(insert(key, UnencodeableEntity(e))) + require.Error(t, err, errCantEncode) + require.NotErrorIs(t, err, storage.ErrNotFound) + }) +} + +func TestUpdateValid(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, []byte{}) + require.NoError(t, err) + return nil + }) + + err := db.Update(update(key, e)) + require.NoError(t, err) + + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + }) +} + +func TestUpdateMissing(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + + err := db.Update(update(key, e)) + require.ErrorIs(t, err, storage.ErrNotFound) + + // ensure nothing was written + _ = db.View(func(tx *badger.Txn) error { + _, err := tx.Get(key) + require.Equal(t, badger.ErrKeyNotFound, err) + return nil + }) + }) +} + +func TestUpdateEncodingError(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + require.NoError(t, err) + return nil + }) + + err := db.Update(update(key, UnencodeableEntity(e))) + require.Error(t, err) + require.NotErrorIs(t, err, storage.ErrNotFound) + + // ensure value did not change + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + }) +} + +func TestUpsertEntry(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + // first upsert an non-existed entry + err := db.Update(insert(key, e)) + require.NoError(t, err) + + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + + // next upsert the value with the same key + newEntity := Entity{ID: 1338} + newVal, _ := msgpack.Marshal(newEntity) + err = db.Update(upsert(key, newEntity)) + require.NoError(t, err) + + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, newVal, act) + }) +} + +func TestRetrieveValid(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + require.NoError(t, err) + return nil + }) + + var act Entity + err := db.View(retrieve(key, &act)) + require.NoError(t, err) + + assert.Equal(t, e, act) + }) +} + +func TestRetrieveMissing(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + key := []byte{0x01, 0x02, 0x03} + + var act Entity + err := db.View(retrieve(key, &act)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) +} + +func TestRetrieveUnencodeable(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + require.NoError(t, err) + return nil + }) + + var act *UnencodeableEntity + err := db.View(retrieve(key, &act)) + require.Error(t, err) + require.NotErrorIs(t, err, storage.ErrNotFound) + }) +} + +// TestExists verifies that `exists` returns correct results in different scenarios. +func TestExists(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("non-existent key", func(t *testing.T) { + key := unittest.RandomBytes(32) + var _exists bool + err := db.View(exists(key, &_exists)) + require.NoError(t, err) + assert.False(t, _exists) + }) + + t.Run("existent key", func(t *testing.T) { + key := unittest.RandomBytes(32) + err := db.Update(insert(key, unittest.RandomBytes(256))) + require.NoError(t, err) + + var _exists bool + err = db.View(exists(key, &_exists)) + require.NoError(t, err) + assert.True(t, _exists) + }) + + t.Run("removed key", func(t *testing.T) { + key := unittest.RandomBytes(32) + // insert, then remove the key + err := db.Update(insert(key, unittest.RandomBytes(256))) + require.NoError(t, err) + err = db.Update(remove(key)) + require.NoError(t, err) + + var _exists bool + err = db.View(exists(key, &_exists)) + require.NoError(t, err) + assert.False(t, _exists) + }) + }) +} + +func TestLookup(t *testing.T) { + expected := []flow.Identifier{ + {0x01}, + {0x02}, + {0x03}, + {0x04}, + } + actual := []flow.Identifier{} + + iterationFunc := lookup(&actual) + + for _, e := range expected { + checkFunc, createFunc, handleFunc := iterationFunc() + assert.True(t, checkFunc([]byte{0x00})) + target := createFunc() + assert.IsType(t, &flow.Identifier{}, target) + + // set the value to target. Need to use reflection here since target is not strongly typed + reflect.ValueOf(target).Elem().Set(reflect.ValueOf(e)) + + assert.NoError(t, handleFunc()) + } + + assert.Equal(t, expected, actual) +} + +func TestIterate(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + keys := [][]byte{{0x00}, {0x12}, {0xf0}, {0xff}} + vals := []bool{false, false, true, true} + expected := []bool{false, true} + + _ = db.Update(func(tx *badger.Txn) error { + for i, key := range keys { + enc, err := msgpack.Marshal(vals[i]) + require.NoError(t, err) + err = tx.Set(key, enc) + require.NoError(t, err) + } + return nil + }) + + actual := make([]bool, 0, len(keys)) + iterationFunc := func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return !bytes.Equal(key, []byte{0x12}) + } + var val bool + create := func() interface{} { + return &val + } + handle := func() error { + actual = append(actual, val) + return nil + } + return check, create, handle + } + + err := db.View(iterate(keys[0], keys[2], iterationFunc)) + require.Nil(t, err) + + assert.Equal(t, expected, actual) + }) +} + +func TestTraverse(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + keys := [][]byte{{0x42, 0x00}, {0xff}, {0x42, 0x56}, {0x00}, {0x42, 0xff}} + vals := []bool{false, false, true, false, true} + expected := []bool{false, true} + + _ = db.Update(func(tx *badger.Txn) error { + for i, key := range keys { + enc, err := msgpack.Marshal(vals[i]) + require.NoError(t, err) + err = tx.Set(key, enc) + require.NoError(t, err) + } + return nil + }) + + actual := make([]bool, 0, len(keys)) + iterationFunc := func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return !bytes.Equal(key, []byte{0x42, 0x56}) + } + var val bool + create := func() interface{} { + return &val + } + handle := func() error { + actual = append(actual, val) + return nil + } + return check, create, handle + } + + err := db.View(traverse([]byte{0x42}, iterationFunc)) + require.Nil(t, err) + + assert.Equal(t, expected, actual) + }) +} + +func TestRemove(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + require.NoError(t, err) + return nil + }) + + t.Run("should be able to remove", func(t *testing.T) { + _ = db.Update(func(txn *badger.Txn) error { + err := remove(key)(txn) + assert.NoError(t, err) + + _, err = txn.Get(key) + assert.ErrorIs(t, err, badger.ErrKeyNotFound) + + return nil + }) + }) + + t.Run("should error when removing non-existing value", func(t *testing.T) { + nonexistantKey := append(key, 0x01) + _ = db.Update(func(txn *badger.Txn) error { + err := remove(nonexistantKey)(txn) + assert.ErrorIs(t, err, storage.ErrNotFound) + assert.Error(t, err) + return nil + }) + }) + }) +} + +func TestRemoveByPrefix(t *testing.T) { + t.Run("should no-op when removing non-existing value", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + assert.NoError(t, err) + return nil + }) + + nonexistantKey := append(key, 0x01) + err := db.Update(removeByPrefix(nonexistantKey)) + assert.NoError(t, err) + + var act Entity + err = db.View(retrieve(key, &act)) + require.NoError(t, err) + + assert.Equal(t, e, act) + }) + }) + + t.Run("should be able to remove", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + assert.NoError(t, err) + return nil + }) + + _ = db.Update(func(txn *badger.Txn) error { + prefix := []byte{0x01, 0x02} + err := removeByPrefix(prefix)(txn) + assert.NoError(t, err) + + _, err = txn.Get(key) + assert.Error(t, err) + assert.IsType(t, badger.ErrKeyNotFound, err) + + return nil + }) + }) + }) + + t.Run("should be able to remove by key", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + _ = db.Update(func(tx *badger.Txn) error { + err := tx.Set(key, val) + assert.NoError(t, err) + return nil + }) + + _ = db.Update(func(txn *badger.Txn) error { + err := removeByPrefix(key)(txn) + assert.NoError(t, err) + + _, err = txn.Get(key) + assert.Error(t, err) + assert.IsType(t, badger.ErrKeyNotFound, err) + + return nil + }) + }) + }) +} + +func TestIterateBoundaries(t *testing.T) { + + // create range of keys covering all boundaries around our start/end values + start := []byte{0x10} + end := []byte{0x20} + keys := [][]byte{ + // before start -> not included in range + {0x09, 0xff}, + // shares prefix with start -> included in range + {0x10, 0x00}, + {0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x10, 0xff}, + {0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + // prefix between start and end -> included in range + {0x11, 0x00}, + {0x19, 0xff}, + // shares prefix with end -> included in range + {0x20, 0x00}, + {0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x20, 0xff}, + {0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + // after end -> not included in range + {0x21, 0x00}, + } + + // set the maximum current DB key range + for _, key := range keys { + if uint32(len(key)) > max { + max = uint32(len(key)) + } + } + + // keys within the expected range + keysInRange := keys[1:11] + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // insert the keys into the database + _ = db.Update(func(tx *badger.Txn) error { + for _, key := range keys { + err := tx.Set(key, []byte{0x00}) + if err != nil { + return err + } + } + return nil + }) + + // define iteration function that simply appends all traversed keys + var found [][]byte + iteration := func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + found = append(found, key) + return false + } + create := func() interface{} { + return nil + } + handle := func() error { + return fmt.Errorf("shouldn't handle anything") + } + return check, create, handle + } + + // iterate forward and check boundaries are included correctly + found = nil + err := db.View(iterate(start, end, iteration)) + for i, f := range found { + t.Logf("forward %d: %x", i, f) + } + require.NoError(t, err, "should iterate forward without error") + assert.ElementsMatch(t, keysInRange, found, "forward iteration should go over correct keys") + + // iterate backward and check boundaries are included correctly + found = nil + err = db.View(iterate(end, start, iteration)) + for i, f := range found { + t.Logf("backward %d: %x", i, f) + } + require.NoError(t, err, "should iterate backward without error") + assert.ElementsMatch(t, keysInRange, found, "backward iteration should go over correct keys") + }) +} + +func TestFindHighestAtOrBelow(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + prefix := []byte("test_prefix") + + type Entity struct { + Value uint64 + } + + entity1 := Entity{Value: 41} + entity2 := Entity{Value: 42} + entity3 := Entity{Value: 43} + + err := db.Update(func(tx *badger.Txn) error { + key := append(prefix, b(uint64(15))...) + val, err := msgpack.Marshal(entity3) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + + key = append(prefix, b(uint64(5))...) + val, err = msgpack.Marshal(entity1) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + + key = append(prefix, b(uint64(10))...) + val, err = msgpack.Marshal(entity2) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + return nil + }) + require.NoError(t, err) + + var entity Entity + + t.Run("target height exists", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 10, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(42), entity.Value) + }) + + t.Run("target height above", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 11, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(42), entity.Value) + }) + + t.Run("target height above highest", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 20, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(43), entity.Value) + }) + + t.Run("target height below lowest", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 4, + &entity)(db.NewTransaction(false)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) + + t.Run("empty prefix", func(t *testing.T) { + err = findHighestAtOrBelow( + []byte{}, + 5, + &entity)(db.NewTransaction(false)) + require.Error(t, err) + require.Contains(t, err.Error(), "prefix must not be empty") + }) + }) +} diff --git a/storage/pebble/operation/computation_result.go b/storage/pebble/operation/computation_result.go new file mode 100644 index 00000000000..22238cc06e5 --- /dev/null +++ b/storage/pebble/operation/computation_result.go @@ -0,0 +1,62 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertComputationResult addes given instance of ComputationResult into local BadgerDB. +func InsertComputationResultUploadStatus(blockID flow.Identifier, + wasUploadCompleted bool) func(*badger.Txn) error { + return insert(makePrefix(codeComputationResults, blockID), wasUploadCompleted) +} + +// UpdateComputationResult updates given existing instance of ComputationResult in local BadgerDB. +func UpdateComputationResultUploadStatus(blockID flow.Identifier, + wasUploadCompleted bool) func(*badger.Txn) error { + return update(makePrefix(codeComputationResults, blockID), wasUploadCompleted) +} + +// UpsertComputationResult upserts given existing instance of ComputationResult in local BadgerDB. +func UpsertComputationResultUploadStatus(blockID flow.Identifier, + wasUploadCompleted bool) func(*badger.Txn) error { + return upsert(makePrefix(codeComputationResults, blockID), wasUploadCompleted) +} + +// RemoveComputationResult removes an instance of ComputationResult with given ID. +func RemoveComputationResultUploadStatus( + blockID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeComputationResults, blockID)) +} + +// GetComputationResult returns stored ComputationResult instance with given ID. +func GetComputationResultUploadStatus(blockID flow.Identifier, + wasUploadCompleted *bool) func(*badger.Txn) error { + return retrieve(makePrefix(codeComputationResults, blockID), wasUploadCompleted) +} + +// GetBlockIDsByStatus returns all IDs of stored ComputationResult instances. +func GetBlockIDsByStatus(blockIDs *[]flow.Identifier, + targetUploadStatus bool) func(*badger.Txn) error { + return traverse(makePrefix(codeComputationResults), func() (checkFunc, createFunc, handleFunc) { + var currKey flow.Identifier + check := func(key []byte) bool { + currKey = flow.HashToID(key[1:]) + return true + } + + var wasUploadCompleted bool + create := func() interface{} { + return &wasUploadCompleted + } + + handle := func() error { + if blockIDs != nil && wasUploadCompleted == targetUploadStatus { + *blockIDs = append(*blockIDs, currKey) + } + return nil + } + return check, create, handle + }) +} diff --git a/storage/pebble/operation/computation_result_test.go b/storage/pebble/operation/computation_result_test.go new file mode 100644 index 00000000000..79336a87964 --- /dev/null +++ b/storage/pebble/operation/computation_result_test.go @@ -0,0 +1,144 @@ +package operation + +import ( + "reflect" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/execution" + "github.com/onflow/flow-go/engine/execution/testutil" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertAndUpdateAndRetrieveComputationResultUpdateStatus(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := testutil.ComputationResultFixture(t) + expectedId := expected.ExecutableBlock.ID() + + t.Run("Update existing ComputationResult", func(t *testing.T) { + // insert as False + testUploadStatusVal := false + + err := db.Update(InsertComputationResultUploadStatus(expectedId, testUploadStatusVal)) + require.NoError(t, err) + + var actualUploadStatus bool + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + require.NoError(t, err) + + assert.Equal(t, testUploadStatusVal, actualUploadStatus) + + // update to True + testUploadStatusVal = true + err = db.Update(UpdateComputationResultUploadStatus(expectedId, testUploadStatusVal)) + require.NoError(t, err) + + // check if value is updated + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + require.NoError(t, err) + + assert.Equal(t, testUploadStatusVal, actualUploadStatus) + }) + + t.Run("Update non-existed ComputationResult", func(t *testing.T) { + testUploadStatusVal := true + randomFlowID := flow.Identifier{} + err := db.Update(UpdateComputationResultUploadStatus(randomFlowID, testUploadStatusVal)) + require.Error(t, err) + require.Equal(t, err, storage.ErrNotFound) + }) + }) +} + +func TestUpsertAndRetrieveComputationResultUpdateStatus(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := testutil.ComputationResultFixture(t) + expectedId := expected.ExecutableBlock.ID() + + t.Run("Upsert ComputationResult", func(t *testing.T) { + // first upsert as false + testUploadStatusVal := false + + err := db.Update(UpsertComputationResultUploadStatus(expectedId, testUploadStatusVal)) + require.NoError(t, err) + + var actualUploadStatus bool + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + require.NoError(t, err) + + assert.Equal(t, testUploadStatusVal, actualUploadStatus) + + // upsert to true + testUploadStatusVal = true + err = db.Update(UpsertComputationResultUploadStatus(expectedId, testUploadStatusVal)) + require.NoError(t, err) + + // check if value is updated + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + require.NoError(t, err) + + assert.Equal(t, testUploadStatusVal, actualUploadStatus) + }) + }) +} + +func TestRemoveComputationResultUploadStatus(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := testutil.ComputationResultFixture(t) + expectedId := expected.ExecutableBlock.ID() + + t.Run("Remove ComputationResult", func(t *testing.T) { + testUploadStatusVal := true + + err := db.Update(InsertComputationResultUploadStatus(expectedId, testUploadStatusVal)) + require.NoError(t, err) + + var actualUploadStatus bool + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + require.NoError(t, err) + + assert.Equal(t, testUploadStatusVal, actualUploadStatus) + + err = db.Update(RemoveComputationResultUploadStatus(expectedId)) + require.NoError(t, err) + + err = db.View(GetComputationResultUploadStatus(expectedId, &actualUploadStatus)) + assert.NotNil(t, err) + }) + }) +} + +func TestListComputationResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := [...]*execution.ComputationResult{ + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), + } + t.Run("List all ComputationResult with status True", func(t *testing.T) { + expectedIDs := make(map[string]bool, 0) + // Store a list of ComputationResult instances first + for _, cr := range expected { + expectedId := cr.ExecutableBlock.ID() + expectedIDs[expectedId.String()] = true + err := db.Update(InsertComputationResultUploadStatus(expectedId, true)) + require.NoError(t, err) + } + + // Get the list of IDs of stored ComputationResult + crIDs := make([]flow.Identifier, 0) + err := db.View(GetBlockIDsByStatus(&crIDs, true)) + require.NoError(t, err) + crIDsStrMap := make(map[string]bool, 0) + for _, crID := range crIDs { + crIDsStrMap[crID.String()] = true + } + + assert.True(t, reflect.DeepEqual(crIDsStrMap, expectedIDs)) + }) + }) +} diff --git a/storage/pebble/operation/dkg.go b/storage/pebble/operation/dkg.go new file mode 100644 index 00000000000..7a468ed9f36 --- /dev/null +++ b/storage/pebble/operation/dkg.go @@ -0,0 +1,69 @@ +package operation + +import ( + "errors" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/encodable" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +// InsertMyBeaconPrivateKey stores the random beacon private key for the given epoch. +// +// CAUTION: This method stores confidential information and should only be +// used in the context of the secrets database. This is enforced in the above +// layer (see storage.DKGState). +// Error returns: storage.ErrAlreadyExists +func InsertMyBeaconPrivateKey(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*badger.Txn) error { + return insert(makePrefix(codeBeaconPrivateKey, epochCounter), info) +} + +// RetrieveMyBeaconPrivateKey retrieves the random beacon private key for the given epoch. +// +// CAUTION: This method stores confidential information and should only be +// used in the context of the secrets database. This is enforced in the above +// layer (see storage.DKGState). +// Error returns: storage.ErrNotFound +func RetrieveMyBeaconPrivateKey(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*badger.Txn) error { + return retrieve(makePrefix(codeBeaconPrivateKey, epochCounter), info) +} + +// InsertDKGStartedForEpoch stores a flag indicating that the DKG has been started for the given epoch. +// Returns: storage.ErrAlreadyExists +// Error returns: storage.ErrAlreadyExists +func InsertDKGStartedForEpoch(epochCounter uint64) func(*badger.Txn) error { + return insert(makePrefix(codeDKGStarted, epochCounter), true) +} + +// RetrieveDKGStartedForEpoch retrieves the DKG started flag for the given epoch. +// If no flag is set, started is set to false and no error is returned. +// No errors expected during normal operation. +func RetrieveDKGStartedForEpoch(epochCounter uint64, started *bool) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + err := retrieve(makePrefix(codeDKGStarted, epochCounter), started)(tx) + if errors.Is(err, storage.ErrNotFound) { + // flag not set - therefore DKG not started + *started = false + return nil + } else if err != nil { + // storage error - set started to zero value + *started = false + return err + } + return nil + } +} + +// InsertDKGEndStateForEpoch stores the DKG end state for the epoch. +// Error returns: storage.ErrAlreadyExists +func InsertDKGEndStateForEpoch(epochCounter uint64, endState flow.DKGEndState) func(*badger.Txn) error { + return insert(makePrefix(codeDKGEnded, epochCounter), endState) +} + +// RetrieveDKGEndStateForEpoch retrieves the DKG end state for the epoch. +// Error returns: storage.ErrNotFound +func RetrieveDKGEndStateForEpoch(epochCounter uint64, endState *flow.DKGEndState) func(*badger.Txn) error { + return retrieve(makePrefix(codeDKGEnded, epochCounter), endState) +} diff --git a/storage/pebble/operation/dkg_test.go b/storage/pebble/operation/dkg_test.go new file mode 100644 index 00000000000..03417e963f6 --- /dev/null +++ b/storage/pebble/operation/dkg_test.go @@ -0,0 +1,100 @@ +package operation + +import ( + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/model/encodable" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestInsertMyDKGPrivateInfo_StoreRetrieve tests writing and reading private DKG info. +func TestMyBeaconPrivateKey_StoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + t.Run("should return error not found when not stored", func(t *testing.T) { + var stored encodable.RandomBeaconPrivKey + err := db.View(RetrieveMyBeaconPrivateKey(1, &stored)) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) + + t.Run("should be able to store and read", func(t *testing.T) { + epochCounter := rand.Uint64() + info := unittest.RandomBeaconPriv() + + // should be able to store + err := db.Update(InsertMyBeaconPrivateKey(epochCounter, info)) + assert.NoError(t, err) + + // should be able to read + var stored encodable.RandomBeaconPrivKey + err = db.View(RetrieveMyBeaconPrivateKey(epochCounter, &stored)) + assert.NoError(t, err) + assert.Equal(t, info, &stored) + + // should fail to read other epoch counter + err = db.View(RetrieveMyBeaconPrivateKey(rand.Uint64(), &stored)) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) + }) +} + +// TestDKGStartedForEpoch tests setting the DKG-started flag. +func TestDKGStartedForEpoch(t *testing.T) { + + t.Run("reading when unset should return false", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var started bool + err := db.View(RetrieveDKGStartedForEpoch(1, &started)) + assert.NoError(t, err) + assert.False(t, started) + }) + }) + + t.Run("should be able to set flag to true", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + epochCounter := rand.Uint64() + + // set the flag, ensure no error + err := db.Update(InsertDKGStartedForEpoch(epochCounter)) + assert.NoError(t, err) + + // read the flag, should be true now + var started bool + err = db.View(RetrieveDKGStartedForEpoch(epochCounter, &started)) + assert.NoError(t, err) + assert.True(t, started) + + // read the flag for a different epoch, should be false + err = db.View(RetrieveDKGStartedForEpoch(epochCounter+1, &started)) + assert.NoError(t, err) + assert.False(t, started) + }) + }) +} + +func TestDKGEndStateForEpoch(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + epochCounter := rand.Uint64() + + // should be able to write end state + endState := flow.DKGEndStateSuccess + err := db.Update(InsertDKGEndStateForEpoch(epochCounter, endState)) + assert.NoError(t, err) + + // should be able to read end state + var readEndState flow.DKGEndState + err = db.View(RetrieveDKGEndStateForEpoch(epochCounter, &readEndState)) + assert.NoError(t, err) + assert.Equal(t, endState, readEndState) + + // attempting to overwrite should error + err = db.Update(InsertDKGEndStateForEpoch(epochCounter, flow.DKGEndStateDKGFailure)) + assert.ErrorIs(t, err, storage.ErrAlreadyExists) + }) +} diff --git a/storage/pebble/operation/epoch.go b/storage/pebble/operation/epoch.go new file mode 100644 index 00000000000..b5fcef7e029 --- /dev/null +++ b/storage/pebble/operation/epoch.go @@ -0,0 +1,75 @@ +package operation + +import ( + "errors" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +func InsertEpochSetup(eventID flow.Identifier, event *flow.EpochSetup) func(*badger.Txn) error { + return insert(makePrefix(codeEpochSetup, eventID), event) +} + +func RetrieveEpochSetup(eventID flow.Identifier, event *flow.EpochSetup) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochSetup, eventID), event) +} + +func InsertEpochCommit(eventID flow.Identifier, event *flow.EpochCommit) func(*badger.Txn) error { + return insert(makePrefix(codeEpochCommit, eventID), event) +} + +func RetrieveEpochCommit(eventID flow.Identifier, event *flow.EpochCommit) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochCommit, eventID), event) +} + +func InsertEpochStatus(blockID flow.Identifier, status *flow.EpochStatus) func(*badger.Txn) error { + return insert(makePrefix(codeBlockEpochStatus, blockID), status) +} + +func RetrieveEpochStatus(blockID flow.Identifier, status *flow.EpochStatus) func(*badger.Txn) error { + return retrieve(makePrefix(codeBlockEpochStatus, blockID), status) +} + +// SetEpochEmergencyFallbackTriggered sets a flag in the DB indicating that +// epoch emergency fallback has been triggered, and the block where it was triggered. +// +// EECC can be triggered in two ways: +// 1. Finalizing the first block past the epoch commitment deadline, when the +// next epoch has not yet been committed (see protocol.Params for more detail) +// 2. Finalizing a fork in which an invalid service event was incorporated. +// +// Calling this function multiple times is a no-op and returns no expected errors. +func SetEpochEmergencyFallbackTriggered(blockID flow.Identifier) func(txn *badger.Txn) error { + return SkipDuplicates(insert(makePrefix(codeEpochEmergencyFallbackTriggered), blockID)) +} + +// RetrieveEpochEmergencyFallbackTriggeredBlockID gets the block ID where epoch +// emergency was triggered. +func RetrieveEpochEmergencyFallbackTriggeredBlockID(blockID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochEmergencyFallbackTriggered), blockID) +} + +// CheckEpochEmergencyFallbackTriggered retrieves the value of the flag +// indicating whether epoch emergency fallback has been triggered. If the key +// is not set, this results in triggered being set to false. +func CheckEpochEmergencyFallbackTriggered(triggered *bool) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + var blockID flow.Identifier + err := RetrieveEpochEmergencyFallbackTriggeredBlockID(&blockID)(tx) + if errors.Is(err, storage.ErrNotFound) { + // flag unset, EECC not triggered + *triggered = false + return nil + } else if err != nil { + // storage error, set triggered to zero value + *triggered = false + return err + } + // flag is set, EECC triggered + *triggered = true + return err + } +} diff --git a/storage/pebble/operation/epoch_test.go b/storage/pebble/operation/epoch_test.go new file mode 100644 index 00000000000..a9d4938e486 --- /dev/null +++ b/storage/pebble/operation/epoch_test.go @@ -0,0 +1,68 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestEpochEmergencyFallback(t *testing.T) { + + // the block ID where EECC was triggered + blockID := unittest.IdentifierFixture() + + t.Run("reading when unset should return false", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + var triggered bool + err := db.View(CheckEpochEmergencyFallbackTriggered(&triggered)) + assert.NoError(t, err) + assert.False(t, triggered) + }) + }) + t.Run("should be able to set flag to true", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + // set the flag, ensure no error + err := db.Update(SetEpochEmergencyFallbackTriggered(blockID)) + assert.NoError(t, err) + + // read the flag, should be true now + var triggered bool + err = db.View(CheckEpochEmergencyFallbackTriggered(&triggered)) + assert.NoError(t, err) + assert.True(t, triggered) + + // read the value of the block ID, should match + var storedBlockID flow.Identifier + err = db.View(RetrieveEpochEmergencyFallbackTriggeredBlockID(&storedBlockID)) + assert.NoError(t, err) + assert.Equal(t, blockID, storedBlockID) + }) + }) + t.Run("setting flag multiple time should have no additional effect", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + // set the flag, ensure no error + err := db.Update(SetEpochEmergencyFallbackTriggered(blockID)) + assert.NoError(t, err) + + // set the flag, should have no error and no effect on state + err = db.Update(SetEpochEmergencyFallbackTriggered(unittest.IdentifierFixture())) + assert.NoError(t, err) + + // read the flag, should be true + var triggered bool + err = db.View(CheckEpochEmergencyFallbackTriggered(&triggered)) + assert.NoError(t, err) + assert.True(t, triggered) + + // read the value of block ID, should equal the FIRST set ID + var storedBlockID flow.Identifier + err = db.View(RetrieveEpochEmergencyFallbackTriggeredBlockID(&storedBlockID)) + assert.NoError(t, err) + assert.Equal(t, blockID, storedBlockID) + }) + }) +} diff --git a/storage/pebble/operation/events.go b/storage/pebble/operation/events.go new file mode 100644 index 00000000000..f49c937c412 --- /dev/null +++ b/storage/pebble/operation/events.go @@ -0,0 +1,115 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func eventPrefix(prefix byte, blockID flow.Identifier, event flow.Event) []byte { + return makePrefix(prefix, blockID, event.TransactionID, event.TransactionIndex, event.EventIndex) +} + +func InsertEvent(blockID flow.Identifier, event flow.Event) func(*badger.Txn) error { + return insert(eventPrefix(codeEvent, blockID, event), event) +} + +func BatchInsertEvent(blockID flow.Identifier, event flow.Event) func(batch *badger.WriteBatch) error { + return batchWrite(eventPrefix(codeEvent, blockID, event), event) +} + +func InsertServiceEvent(blockID flow.Identifier, event flow.Event) func(*badger.Txn) error { + return insert(eventPrefix(codeServiceEvent, blockID, event), event) +} + +func BatchInsertServiceEvent(blockID flow.Identifier, event flow.Event) func(batch *badger.WriteBatch) error { + return batchWrite(eventPrefix(codeServiceEvent, blockID, event), event) +} + +func RetrieveEvents(blockID flow.Identifier, transactionID flow.Identifier, events *[]flow.Event) func(*badger.Txn) error { + iterationFunc := eventIterationFunc(events) + return traverse(makePrefix(codeEvent, blockID, transactionID), iterationFunc) +} + +func LookupEventsByBlockID(blockID flow.Identifier, events *[]flow.Event) func(*badger.Txn) error { + iterationFunc := eventIterationFunc(events) + return traverse(makePrefix(codeEvent, blockID), iterationFunc) +} + +func LookupServiceEventsByBlockID(blockID flow.Identifier, events *[]flow.Event) func(*badger.Txn) error { + iterationFunc := eventIterationFunc(events) + return traverse(makePrefix(codeServiceEvent, blockID), iterationFunc) +} + +func LookupEventsByBlockIDEventType(blockID flow.Identifier, eventType flow.EventType, events *[]flow.Event) func(*badger.Txn) error { + iterationFunc := eventFilterIterationFunc(events, eventType) + return traverse(makePrefix(codeEvent, blockID), iterationFunc) +} + +func RemoveServiceEventsByBlockID(blockID flow.Identifier) func(*badger.Txn) error { + return removeByPrefix(makePrefix(codeServiceEvent, blockID)) +} + +// BatchRemoveServiceEventsByBlockID removes all service events for the given blockID. +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func BatchRemoveServiceEventsByBlockID(blockID flow.Identifier, batch *badger.WriteBatch) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + return batchRemoveByPrefix(makePrefix(codeServiceEvent, blockID))(txn, batch) + } +} + +func RemoveEventsByBlockID(blockID flow.Identifier) func(*badger.Txn) error { + return removeByPrefix(makePrefix(codeEvent, blockID)) +} + +// BatchRemoveEventsByBlockID removes all events for the given blockID. +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func BatchRemoveEventsByBlockID(blockID flow.Identifier, batch *badger.WriteBatch) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + return batchRemoveByPrefix(makePrefix(codeEvent, blockID))(txn, batch) + } + +} + +// eventIterationFunc returns an in iteration function which returns all events found during traversal or iteration +func eventIterationFunc(events *[]flow.Event) func() (checkFunc, createFunc, handleFunc) { + return func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return true + } + var val flow.Event + create := func() interface{} { + return &val + } + handle := func() error { + *events = append(*events, val) + return nil + } + return check, create, handle + } +} + +// eventFilterIterationFunc returns an iteration function which filters the result by the given event type in the handleFunc +func eventFilterIterationFunc(events *[]flow.Event, eventType flow.EventType) func() (checkFunc, createFunc, handleFunc) { + return func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return true + } + var val flow.Event + create := func() interface{} { + return &val + } + handle := func() error { + // filter out all events not of type eventType + if val.Type == eventType { + *events = append(*events, val) + } + return nil + } + return check, create, handle + } +} diff --git a/storage/pebble/operation/events_test.go b/storage/pebble/operation/events_test.go new file mode 100644 index 00000000000..9896c02fd69 --- /dev/null +++ b/storage/pebble/operation/events_test.go @@ -0,0 +1,128 @@ +package operation + +import ( + "bytes" + "testing" + + "golang.org/x/exp/slices" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestRetrieveEventByBlockIDTxID tests event insertion, event retrieval by block id, block id and transaction id, +// and block id and event type +func TestRetrieveEventByBlockIDTxID(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // create block ids, transaction ids and event types slices + blockIDs := []flow.Identifier{flow.HashToID([]byte{0x01}), flow.HashToID([]byte{0x02})} + txIDs := []flow.Identifier{flow.HashToID([]byte{0x11}), flow.HashToID([]byte{0x12})} + eTypes := []flow.EventType{flow.EventAccountCreated, flow.EventAccountUpdated} + + // create map of block id to event, tx id to event and event type to event + blockMap := make(map[string][]flow.Event) + txMap := make(map[string][]flow.Event) + typeMap := make(map[string][]flow.Event) + + // initialize the maps and the db + for _, b := range blockIDs { + + bEvents := make([]flow.Event, 0) + + // all blocks share the same transactions + for i, tx := range txIDs { + + tEvents := make([]flow.Event, 0) + + // create one event for each possible event type + for j, etype := range eTypes { + + eEvents := make([]flow.Event, 0) + + event := unittest.EventFixture(etype, uint32(i), uint32(j), tx, 0) + + // insert event into the db + err := db.Update(InsertEvent(b, event)) + require.Nil(t, err) + + // update event arrays in the maps + bEvents = append(bEvents, event) + tEvents = append(tEvents, event) + eEvents = append(eEvents, event) + + key := b.String() + "_" + string(etype) + if _, ok := typeMap[key]; ok { + typeMap[key] = append(typeMap[key], eEvents...) + } else { + typeMap[key] = eEvents + } + } + txMap[b.String()+"_"+tx.String()] = tEvents + } + blockMap[b.String()] = bEvents + } + + assertFunc := func(err error, expected []flow.Event, actual []flow.Event) { + require.NoError(t, err) + sortEvent(expected) + sortEvent(actual) + require.Equal(t, expected, actual) + } + + t.Run("retrieve events by Block ID", func(t *testing.T) { + for _, b := range blockIDs { + var actualEvents = make([]flow.Event, 0) + + // lookup events by block id + err := db.View(LookupEventsByBlockID(b, &actualEvents)) + + expectedEvents := blockMap[b.String()] + assertFunc(err, expectedEvents, actualEvents) + } + }) + + t.Run("retrieve events by block ID and transaction ID", func(t *testing.T) { + for _, b := range blockIDs { + for _, t := range txIDs { + var actualEvents = make([]flow.Event, 0) + + //lookup events by block id and transaction id + err := db.View(RetrieveEvents(b, t, &actualEvents)) + + expectedEvents := txMap[b.String()+"_"+t.String()] + assertFunc(err, expectedEvents, actualEvents) + } + } + }) + + t.Run("retrieve events by block ID and event type", func(t *testing.T) { + for _, b := range blockIDs { + for _, et := range eTypes { + var actualEvents = make([]flow.Event, 0) + + //lookup events by block id and transaction id + err := db.View(LookupEventsByBlockIDEventType(b, et, &actualEvents)) + + expectedEvents := typeMap[b.String()+"_"+string(et)] + assertFunc(err, expectedEvents, actualEvents) + } + } + }) + }) +} + +// Event retrieval does not guarantee any order, +// Hence, we a sort the events for comparing the expected and actual events. +func sortEvent(events []flow.Event) { + slices.SortFunc(events, func(i, j flow.Event) int { + tComp := bytes.Compare(i.TransactionID[:], j.TransactionID[:]) + if tComp != 0 { + return tComp + } + return int(i.EventIndex) - int(j.EventIndex) + }) +} diff --git a/storage/pebble/operation/guarantees.go b/storage/pebble/operation/guarantees.go new file mode 100644 index 00000000000..cfefead5f5b --- /dev/null +++ b/storage/pebble/operation/guarantees.go @@ -0,0 +1,23 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func InsertGuarantee(collID flow.Identifier, guarantee *flow.CollectionGuarantee) func(*badger.Txn) error { + return insert(makePrefix(codeGuarantee, collID), guarantee) +} + +func RetrieveGuarantee(collID flow.Identifier, guarantee *flow.CollectionGuarantee) func(*badger.Txn) error { + return retrieve(makePrefix(codeGuarantee, collID), guarantee) +} + +func IndexPayloadGuarantees(blockID flow.Identifier, guarIDs []flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codePayloadGuarantees, blockID), guarIDs) +} + +func LookupPayloadGuarantees(blockID flow.Identifier, guarIDs *[]flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codePayloadGuarantees, blockID), guarIDs) +} diff --git a/storage/pebble/operation/guarantees_test.go b/storage/pebble/operation/guarantees_test.go new file mode 100644 index 00000000000..3045799db58 --- /dev/null +++ b/storage/pebble/operation/guarantees_test.go @@ -0,0 +1,122 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestGuaranteeInsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + g := unittest.CollectionGuaranteeFixture() + + err := db.Update(InsertGuarantee(g.CollectionID, g)) + require.Nil(t, err) + + var retrieved flow.CollectionGuarantee + err = db.View(RetrieveGuarantee(g.CollectionID, &retrieved)) + require.NoError(t, err) + + assert.Equal(t, g, &retrieved) + }) +} + +func TestIndexGuaranteedCollectionByBlockHashInsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blockID := flow.Identifier{0x10} + collID1 := flow.Identifier{0x01} + collID2 := flow.Identifier{0x02} + guarantees := []*flow.CollectionGuarantee{ + {CollectionID: collID1, Signature: crypto.Signature{0x10}}, + {CollectionID: collID2, Signature: crypto.Signature{0x20}}, + } + expected := flow.GetIDs(guarantees) + + err := db.Update(func(tx *badger.Txn) error { + for _, guarantee := range guarantees { + if err := InsertGuarantee(guarantee.ID(), guarantee)(tx); err != nil { + return err + } + } + if err := IndexPayloadGuarantees(blockID, expected)(tx); err != nil { + return err + } + return nil + }) + require.Nil(t, err) + + var actual []flow.Identifier + err = db.View(LookupPayloadGuarantees(blockID, &actual)) + require.Nil(t, err) + + assert.Equal(t, []flow.Identifier{collID1, collID2}, actual) + }) +} + +func TestIndexGuaranteedCollectionByBlockHashMultipleBlocks(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blockID1 := flow.Identifier{0x10} + blockID2 := flow.Identifier{0x20} + collID1 := flow.Identifier{0x01} + collID2 := flow.Identifier{0x02} + collID3 := flow.Identifier{0x03} + collID4 := flow.Identifier{0x04} + set1 := []*flow.CollectionGuarantee{ + {CollectionID: collID1, Signature: crypto.Signature{0x1}}, + } + set2 := []*flow.CollectionGuarantee{ + {CollectionID: collID2, Signature: crypto.Signature{0x2}}, + {CollectionID: collID3, Signature: crypto.Signature{0x3}}, + {CollectionID: collID4, Signature: crypto.Signature{0x1}}, + } + ids1 := flow.GetIDs(set1) + ids2 := flow.GetIDs(set2) + + // insert block 1 + err := db.Update(func(tx *badger.Txn) error { + for _, guarantee := range set1 { + if err := InsertGuarantee(guarantee.CollectionID, guarantee)(tx); err != nil { + return err + } + } + if err := IndexPayloadGuarantees(blockID1, ids1)(tx); err != nil { + return err + } + return nil + }) + require.Nil(t, err) + + // insert block 2 + err = db.Update(func(tx *badger.Txn) error { + for _, guarantee := range set2 { + if err := InsertGuarantee(guarantee.CollectionID, guarantee)(tx); err != nil { + return err + } + } + if err := IndexPayloadGuarantees(blockID2, ids2)(tx); err != nil { + return err + } + return nil + }) + require.Nil(t, err) + + t.Run("should retrieve collections for block", func(t *testing.T) { + var actual1 []flow.Identifier + err = db.View(LookupPayloadGuarantees(blockID1, &actual1)) + assert.NoError(t, err) + assert.ElementsMatch(t, []flow.Identifier{collID1}, actual1) + + // get block 2 + var actual2 []flow.Identifier + err = db.View(LookupPayloadGuarantees(blockID2, &actual2)) + assert.NoError(t, err) + assert.Equal(t, []flow.Identifier{collID2, collID3, collID4}, actual2) + }) + }) +} diff --git a/storage/pebble/operation/headers.go b/storage/pebble/operation/headers.go new file mode 100644 index 00000000000..bd1c377cc16 --- /dev/null +++ b/storage/pebble/operation/headers.go @@ -0,0 +1,77 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func InsertHeader(headerID flow.Identifier, header *flow.Header) func(*badger.Txn) error { + return insert(makePrefix(codeHeader, headerID), header) +} + +func RetrieveHeader(blockID flow.Identifier, header *flow.Header) func(*badger.Txn) error { + return retrieve(makePrefix(codeHeader, blockID), header) +} + +// IndexBlockHeight indexes the height of a block. It should only be called on +// finalized blocks. +func IndexBlockHeight(height uint64, blockID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeHeightToBlock, height), blockID) +} + +// LookupBlockHeight retrieves finalized blocks by height. +func LookupBlockHeight(height uint64, blockID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeHeightToBlock, height), blockID) +} + +// BlockExists checks whether the block exists in the database. +// No errors are expected during normal operation. +func BlockExists(blockID flow.Identifier, blockExists *bool) func(*badger.Txn) error { + return exists(makePrefix(codeHeader, blockID), blockExists) +} + +func InsertExecutedBlock(blockID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeExecutedBlock), blockID) +} + +func UpdateExecutedBlock(blockID flow.Identifier) func(*badger.Txn) error { + return update(makePrefix(codeExecutedBlock), blockID) +} + +func RetrieveExecutedBlock(blockID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeExecutedBlock), blockID) +} + +// IndexCollectionBlock indexes a block by a collection within that block. +func IndexCollectionBlock(collID flow.Identifier, blockID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeCollectionBlock, collID), blockID) +} + +// LookupCollectionBlock looks up a block by a collection within that block. +func LookupCollectionBlock(collID flow.Identifier, blockID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeCollectionBlock, collID), blockID) +} + +// FindHeaders iterates through all headers, calling `filter` on each, and adding +// them to the `found` slice if `filter` returned true +func FindHeaders(filter func(header *flow.Header) bool, found *[]flow.Header) func(*badger.Txn) error { + return traverse(makePrefix(codeHeader), func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return true + } + var val flow.Header + create := func() interface{} { + return &val + } + handle := func() error { + if filter(&val) { + *found = append(*found, val) + } + return nil + } + return check, create, handle + }) +} diff --git a/storage/pebble/operation/headers_test.go b/storage/pebble/operation/headers_test.go new file mode 100644 index 00000000000..089ecea3848 --- /dev/null +++ b/storage/pebble/operation/headers_test.go @@ -0,0 +1,74 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + "time" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestHeaderInsertCheckRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := &flow.Header{ + View: 1337, + Timestamp: time.Now().UTC(), + ParentID: flow.Identifier{0x11}, + PayloadHash: flow.Identifier{0x22}, + ParentVoterIndices: []byte{0x44}, + ParentVoterSigData: []byte{0x88}, + ProposerID: flow.Identifier{0x33}, + ProposerSigData: crypto.Signature{0x77}, + } + blockID := expected.ID() + + err := db.Update(InsertHeader(expected.ID(), expected)) + require.Nil(t, err) + + var actual flow.Header + err = db.View(RetrieveHeader(blockID, &actual)) + require.Nil(t, err) + + assert.Equal(t, *expected, actual) + }) +} + +func TestHeaderIDIndexByCollectionID(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + headerID := unittest.IdentifierFixture() + collectionID := unittest.IdentifierFixture() + + err := db.Update(IndexCollectionBlock(collectionID, headerID)) + require.Nil(t, err) + + actualID := &flow.Identifier{} + err = db.View(LookupCollectionBlock(collectionID, actualID)) + require.Nil(t, err) + assert.Equal(t, headerID, *actualID) + }) +} + +func TestBlockHeightIndexLookup(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + height := uint64(1337) + expected := flow.Identifier{0x01, 0x02, 0x03} + + err := db.Update(IndexBlockHeight(height, expected)) + require.Nil(t, err) + + var actual flow.Identifier + err = db.View(LookupBlockHeight(height, &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/operation/heights.go b/storage/pebble/operation/heights.go new file mode 100644 index 00000000000..0c6573ab24c --- /dev/null +++ b/storage/pebble/operation/heights.go @@ -0,0 +1,93 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "github.com/dgraph-io/badger/v2" +) + +func InsertRootHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeFinalizedRootHeight), height) +} + +func RetrieveRootHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeFinalizedRootHeight), height) +} + +func InsertSealedRootHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeSealedRootHeight), height) +} + +func RetrieveSealedRootHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeSealedRootHeight), height) +} + +func InsertFinalizedHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeFinalizedHeight), height) +} + +func UpdateFinalizedHeight(height uint64) func(*badger.Txn) error { + return update(makePrefix(codeFinalizedHeight), height) +} + +func RetrieveFinalizedHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeFinalizedHeight), height) +} + +func InsertSealedHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeSealedHeight), height) +} + +func UpdateSealedHeight(height uint64) func(*badger.Txn) error { + return update(makePrefix(codeSealedHeight), height) +} + +func RetrieveSealedHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeSealedHeight), height) +} + +// InsertEpochFirstHeight inserts the height of the first block in the given epoch. +// The first block of an epoch E is the finalized block with view >= E.FirstView. +// Although we don't store the final height of an epoch, it can be inferred from this index. +// Returns storage.ErrAlreadyExists if the height has already been indexed. +func InsertEpochFirstHeight(epoch, height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeEpochFirstHeight, epoch), height) +} + +// RetrieveEpochFirstHeight retrieves the height of the first block in the given epoch. +// Returns storage.ErrNotFound if the first block of the epoch has not yet been finalized. +func RetrieveEpochFirstHeight(epoch uint64, height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochFirstHeight, epoch), height) +} + +// RetrieveEpochLastHeight retrieves the height of the last block in the given epoch. +// It's a more readable, but equivalent query to RetrieveEpochFirstHeight when interested in the last height of an epoch. +// Returns storage.ErrNotFound if the first block of the epoch has not yet been finalized. +func RetrieveEpochLastHeight(epoch uint64, height *uint64) func(*badger.Txn) error { + var nextEpochFirstHeight uint64 + return func(tx *badger.Txn) error { + if err := retrieve(makePrefix(codeEpochFirstHeight, epoch+1), &nextEpochFirstHeight)(tx); err != nil { + return err + } + *height = nextEpochFirstHeight - 1 + return nil + } +} + +// InsertLastCompleteBlockHeightIfNotExists inserts the last full block height if it is not already set. +// Calling this function multiple times is a no-op and returns no expected errors. +func InsertLastCompleteBlockHeightIfNotExists(height uint64) func(*badger.Txn) error { + return SkipDuplicates(InsertLastCompleteBlockHeight(height)) +} + +func InsertLastCompleteBlockHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeLastCompleteBlockHeight), height) +} + +func UpdateLastCompleteBlockHeight(height uint64) func(*badger.Txn) error { + return update(makePrefix(codeLastCompleteBlockHeight), height) +} + +func RetrieveLastCompleteBlockHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeLastCompleteBlockHeight), height) +} diff --git a/storage/pebble/operation/heights_test.go b/storage/pebble/operation/heights_test.go new file mode 100644 index 00000000000..5cfa1a77099 --- /dev/null +++ b/storage/pebble/operation/heights_test.go @@ -0,0 +1,140 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestFinalizedInsertUpdateRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height := uint64(1337) + + err := db.Update(InsertFinalizedHeight(height)) + require.NoError(t, err) + + var retrieved uint64 + err = db.View(RetrieveFinalizedHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + + height = 9999 + err = db.Update(UpdateFinalizedHeight(height)) + require.NoError(t, err) + + err = db.View(RetrieveFinalizedHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + }) +} + +func TestSealedInsertUpdateRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height := uint64(1337) + + err := db.Update(InsertSealedHeight(height)) + require.NoError(t, err) + + var retrieved uint64 + err = db.View(RetrieveSealedHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + + height = 9999 + err = db.Update(UpdateSealedHeight(height)) + require.NoError(t, err) + + err = db.View(RetrieveSealedHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + }) +} + +func TestEpochFirstBlockIndex_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height := rand.Uint64() + epoch := rand.Uint64() + + // retrieve when empty errors + var retrieved uint64 + err := db.View(RetrieveEpochFirstHeight(epoch, &retrieved)) + require.ErrorIs(t, err, storage.ErrNotFound) + + // can insert + err = db.Update(InsertEpochFirstHeight(epoch, height)) + require.NoError(t, err) + + // can retrieve + err = db.View(RetrieveEpochFirstHeight(epoch, &retrieved)) + require.NoError(t, err) + assert.Equal(t, retrieved, height) + + // retrieve non-existent key errors + err = db.View(RetrieveEpochFirstHeight(epoch+1, &retrieved)) + require.ErrorIs(t, err, storage.ErrNotFound) + + // insert existent key errors + err = db.Update(InsertEpochFirstHeight(epoch, height)) + require.ErrorIs(t, err, storage.ErrAlreadyExists) + }) +} + +func TestLastCompleteBlockHeightInsertUpdateRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height := uint64(1337) + + err := db.Update(InsertLastCompleteBlockHeight(height)) + require.NoError(t, err) + + var retrieved uint64 + err = db.View(RetrieveLastCompleteBlockHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + + height = 9999 + err = db.Update(UpdateLastCompleteBlockHeight(height)) + require.NoError(t, err) + + err = db.View(RetrieveLastCompleteBlockHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height) + }) +} + +func TestLastCompleteBlockHeightInsertIfNotExists(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height1 := uint64(1337) + + err := db.Update(InsertLastCompleteBlockHeightIfNotExists(height1)) + require.NoError(t, err) + + var retrieved uint64 + err = db.View(RetrieveLastCompleteBlockHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height1) + + height2 := uint64(9999) + err = db.Update(InsertLastCompleteBlockHeightIfNotExists(height2)) + require.NoError(t, err) + + err = db.View(RetrieveLastCompleteBlockHeight(&retrieved)) + require.NoError(t, err) + + assert.Equal(t, retrieved, height1) + }) +} diff --git a/storage/pebble/operation/init.go b/storage/pebble/operation/init.go new file mode 100644 index 00000000000..7f3fff228c1 --- /dev/null +++ b/storage/pebble/operation/init.go @@ -0,0 +1,88 @@ +package operation + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/storage" +) + +// marker used to denote a database type +type dbTypeMarker int + +func (marker dbTypeMarker) String() string { + return [...]string{ + "dbMarkerPublic", + "dbMarkerSecret", + }[marker] +} + +const ( + // dbMarkerPublic denotes the public database + dbMarkerPublic dbTypeMarker = iota + // dbMarkerSecret denotes the secrets database + dbMarkerSecret +) + +func InsertPublicDBMarker(txn *badger.Txn) error { + return insertDBTypeMarker(dbMarkerPublic)(txn) +} + +func InsertSecretDBMarker(txn *badger.Txn) error { + return insertDBTypeMarker(dbMarkerSecret)(txn) +} + +func EnsurePublicDB(db *badger.DB) error { + return ensureDBWithType(db, dbMarkerPublic) +} + +func EnsureSecretDB(db *badger.DB) error { + return ensureDBWithType(db, dbMarkerSecret) +} + +// insertDBTypeMarker inserts a database type marker if none exists. If a marker +// already exists in the database, this function will return an error if the +// marker does not match the argument, or return nil if it matches. +func insertDBTypeMarker(marker dbTypeMarker) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + var storedMarker dbTypeMarker + err := retrieveDBType(&storedMarker)(txn) + if err != nil && !errors.Is(err, storage.ErrNotFound) { + return fmt.Errorf("could not check db type marker: %w", err) + } + + // we retrieved a marker from storage + if err == nil { + // the marker in storage does not match - error + if storedMarker != marker { + return fmt.Errorf("could not store db type marker - inconsistent marker already stored (expected: %s, actual: %s)", marker, storedMarker) + } + // the marker is already in storage - we're done + return nil + } + + // no marker in storage, insert it + return insert(makePrefix(codeDBType), marker)(txn) + } +} + +// ensureDBWithType ensures the given database has been initialized with the +// given database type marker. If the given database has not been initialized +// with any marker, or with a different marker than expected, returns an error. +func ensureDBWithType(db *badger.DB, expectedMarker dbTypeMarker) error { + var actualMarker dbTypeMarker + err := db.View(retrieveDBType(&actualMarker)) + if err != nil { + return fmt.Errorf("could not get db type: %w", err) + } + if actualMarker != expectedMarker { + return fmt.Errorf("wrong db type (expected: %s, actual: %s)", expectedMarker, actualMarker) + } + return nil +} + +func retrieveDBType(marker *dbTypeMarker) func(*badger.Txn) error { + return retrieve(makePrefix(codeDBType), marker) +} diff --git a/storage/pebble/operation/init_test.go b/storage/pebble/operation/init_test.go new file mode 100644 index 00000000000..c589e22dadb --- /dev/null +++ b/storage/pebble/operation/init_test.go @@ -0,0 +1,76 @@ +package operation_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertRetrieveDBTypeMarker(t *testing.T) { + t.Run("should insert and ensure type marker", func(t *testing.T) { + t.Run("public", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // can insert db marker to empty DB + err := db.Update(operation.InsertPublicDBMarker) + require.NoError(t, err) + // can insert db marker twice + err = db.Update(operation.InsertPublicDBMarker) + require.NoError(t, err) + // ensure correct db type succeeds + err = operation.EnsurePublicDB(db) + require.NoError(t, err) + // ensure other db type fails + err = operation.EnsureSecretDB(db) + require.Error(t, err) + }) + }) + + t.Run("secret", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // can insert db marker to empty DB + err := db.Update(operation.InsertSecretDBMarker) + require.NoError(t, err) + // can insert db marker twice + err = db.Update(operation.InsertSecretDBMarker) + require.NoError(t, err) + // ensure correct db type succeeds + err = operation.EnsureSecretDB(db) + require.NoError(t, err) + // ensure other db type fails + err = operation.EnsurePublicDB(db) + require.Error(t, err) + }) + }) + }) + + t.Run("should fail to insert different db marker to non-empty db", func(t *testing.T) { + t.Run("public", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // can insert db marker to empty DB + err := db.Update(operation.InsertPublicDBMarker) + require.NoError(t, err) + // inserting a different marker should fail + err = db.Update(operation.InsertSecretDBMarker) + require.Error(t, err) + }) + }) + t.Run("secret", func(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + // can insert db marker to empty DB + err := db.Update(operation.InsertSecretDBMarker) + require.NoError(t, err) + // inserting a different marker should fail + err = db.Update(operation.InsertPublicDBMarker) + require.Error(t, err) + }) + }) + }) +} diff --git a/storage/pebble/operation/interactions.go b/storage/pebble/operation/interactions.go new file mode 100644 index 00000000000..952b2f7a188 --- /dev/null +++ b/storage/pebble/operation/interactions.go @@ -0,0 +1,25 @@ +package operation + +import ( + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" + + "github.com/dgraph-io/badger/v2" +) + +func InsertExecutionStateInteractions( + blockID flow.Identifier, + executionSnapshots []*snapshot.ExecutionSnapshot, +) func(*badger.Txn) error { + return insert( + makePrefix(codeExecutionStateInteractions, blockID), + executionSnapshots) +} + +func RetrieveExecutionStateInteractions( + blockID flow.Identifier, + executionSnapshots *[]*snapshot.ExecutionSnapshot, +) func(*badger.Txn) error { + return retrieve( + makePrefix(codeExecutionStateInteractions, blockID), executionSnapshots) +} diff --git a/storage/pebble/operation/interactions_test.go b/storage/pebble/operation/interactions_test.go new file mode 100644 index 00000000000..b976a2dafd1 --- /dev/null +++ b/storage/pebble/operation/interactions_test.go @@ -0,0 +1,62 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestStateInteractionsInsertCheckRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + id1 := flow.NewRegisterID( + flow.BytesToAddress([]byte("\x89krg\u007fBN\x1d\xf5\xfb\xb8r\xbc4\xbd\x98ռ\xf1\xd0twU\xbf\x16N\xb4?,\xa0&;")), + "") + id2 := flow.NewRegisterID(flow.BytesToAddress([]byte{2}), "") + id3 := flow.NewRegisterID(flow.BytesToAddress([]byte{3}), "") + + executionSnapshot := &snapshot.ExecutionSnapshot{ + ReadSet: map[flow.RegisterID]struct{}{ + id2: {}, + id3: {}, + }, + WriteSet: map[flow.RegisterID]flow.RegisterValue{ + id1: []byte("zażółć gęślą jaźń"), + id2: []byte("c"), + }, + } + + interactions := []*snapshot.ExecutionSnapshot{ + executionSnapshot, + {}, + } + + blockID := unittest.IdentifierFixture() + + err := db.Update(InsertExecutionStateInteractions(blockID, interactions)) + require.Nil(t, err) + + var readInteractions []*snapshot.ExecutionSnapshot + + err = db.View(RetrieveExecutionStateInteractions(blockID, &readInteractions)) + require.NoError(t, err) + + assert.Equal(t, interactions, readInteractions) + assert.Equal( + t, + executionSnapshot.WriteSet, + readInteractions[0].WriteSet) + assert.Equal( + t, + executionSnapshot.ReadSet, + readInteractions[0].ReadSet) + }) +} diff --git a/storage/pebble/operation/jobs.go b/storage/pebble/operation/jobs.go new file mode 100644 index 00000000000..0f9eb3166ad --- /dev/null +++ b/storage/pebble/operation/jobs.go @@ -0,0 +1,43 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func RetrieveJobLatestIndex(queue string, index *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeJobQueuePointer, queue), index) +} + +func InitJobLatestIndex(queue string, index uint64) func(*badger.Txn) error { + return insert(makePrefix(codeJobQueuePointer, queue), index) +} + +func SetJobLatestIndex(queue string, index uint64) func(*badger.Txn) error { + return update(makePrefix(codeJobQueuePointer, queue), index) +} + +// RetrieveJobAtIndex returns the entity at the given index +func RetrieveJobAtIndex(queue string, index uint64, entity *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeJobQueue, queue, index), entity) +} + +// InsertJobAtIndex insert an entity ID at the given index +func InsertJobAtIndex(queue string, index uint64, entity flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeJobQueue, queue, index), entity) +} + +// RetrieveProcessedIndex returns the processed index for a job consumer +func RetrieveProcessedIndex(jobName string, processed *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeJobConsumerProcessed, jobName), processed) +} + +func InsertProcessedIndex(jobName string, processed uint64) func(*badger.Txn) error { + return insert(makePrefix(codeJobConsumerProcessed, jobName), processed) +} + +// SetProcessedIndex updates the processed index for a job consumer with given index +func SetProcessedIndex(jobName string, processed uint64) func(*badger.Txn) error { + return update(makePrefix(codeJobConsumerProcessed, jobName), processed) +} diff --git a/storage/pebble/operation/max.go b/storage/pebble/operation/max.go new file mode 100644 index 00000000000..754e2e9bcb7 --- /dev/null +++ b/storage/pebble/operation/max.go @@ -0,0 +1,57 @@ +package operation + +import ( + "encoding/binary" + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/storage" +) + +// maxKey is the biggest allowed key size in badger +const maxKey = 65000 + +// max holds the maximum length of keys in the database; in order to optimize +// the end prefix of iteration, we need to know how many `0xff` bytes to add. +var max uint32 + +// we initialize max to maximum size, to detect if it wasn't set yet +func init() { + max = maxKey +} + +// InitMax retrieves the maximum key length to have it internally in the +// package after restarting. +// No errors are expected during normal operation. +func InitMax(tx *badger.Txn) error { + key := makePrefix(codeMax) + item, err := tx.Get(key) + if errors.Is(err, badger.ErrKeyNotFound) { // just keep zero value as default + max = 0 + return nil + } + if err != nil { + return fmt.Errorf("could not get max: %w", err) + } + _ = item.Value(func(val []byte) error { + max = binary.LittleEndian.Uint32(val) + return nil + }) + return nil +} + +// SetMax sets the value for the maximum key length used for efficient iteration. +// No errors are expected during normal operation. +func SetMax(tx storage.Transaction) error { + key := makePrefix(codeMax) + val := make([]byte, 4) + binary.LittleEndian.PutUint32(val, max) + err := tx.Set(key, val) + if err != nil { + return irrecoverable.NewExceptionf("could not set max: %w", err) + } + return nil +} diff --git a/storage/pebble/operation/modifiers.go b/storage/pebble/operation/modifiers.go new file mode 100644 index 00000000000..3965b5d204c --- /dev/null +++ b/storage/pebble/operation/modifiers.go @@ -0,0 +1,57 @@ +package operation + +import ( + "errors" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +func SkipDuplicates(op func(*badger.Txn) error) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + err := op(tx) + if errors.Is(err, storage.ErrAlreadyExists) { + metrics.GetStorageCollector().SkipDuplicate() + return nil + } + return err + } +} + +func SkipNonExist(op func(*badger.Txn) error) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + err := op(tx) + if errors.Is(err, badger.ErrKeyNotFound) { + return nil + } + if errors.Is(err, storage.ErrNotFound) { + return nil + } + return err + } +} + +func RetryOnConflict(action func(func(*badger.Txn) error) error, op func(tx *badger.Txn) error) error { + for { + err := action(op) + if errors.Is(err, badger.ErrConflict) { + metrics.GetStorageCollector().RetryOnConflict() + continue + } + return err + } +} + +func RetryOnConflictTx(db *badger.DB, action func(*badger.DB, func(*transaction.Tx) error) error, op func(*transaction.Tx) error) error { + for { + err := action(db, op) + if errors.Is(err, badger.ErrConflict) { + metrics.GetStorageCollector().RetryOnConflict() + continue + } + return err + } +} diff --git a/storage/pebble/operation/modifiers_test.go b/storage/pebble/operation/modifiers_test.go new file mode 100644 index 00000000000..ffeda8440ad --- /dev/null +++ b/storage/pebble/operation/modifiers_test.go @@ -0,0 +1,127 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "errors" + "fmt" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/msgpack/v4" + + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestSkipDuplicates(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + e := Entity{ID: 1337} + key := []byte{0x01, 0x02, 0x03} + val, _ := msgpack.Marshal(e) + + // persist first time + err := db.Update(insert(key, e)) + require.NoError(t, err) + + e2 := Entity{ID: 1338} + + // persist again + err = db.Update(SkipDuplicates(insert(key, e2))) + require.NoError(t, err) + + // ensure old value is still used + var act []byte + _ = db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + require.NoError(t, err) + act, err = item.ValueCopy(nil) + require.NoError(t, err) + return nil + }) + + assert.Equal(t, val, act) + }) +} + +func TestRetryOnConflict(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("good op", func(t *testing.T) { + goodOp := func(*badger.Txn) error { + return nil + } + err := RetryOnConflict(db.Update, goodOp) + require.NoError(t, err) + }) + + t.Run("conflict op should be retried", func(t *testing.T) { + n := 0 + conflictOp := func(*badger.Txn) error { + n++ + if n > 3 { + return nil + } + return badger.ErrConflict + } + err := RetryOnConflict(db.Update, conflictOp) + require.NoError(t, err) + }) + + t.Run("wrapped conflict op should be retried", func(t *testing.T) { + n := 0 + conflictOp := func(*badger.Txn) error { + n++ + if n > 3 { + return nil + } + return fmt.Errorf("wrap error: %w", badger.ErrConflict) + } + err := RetryOnConflict(db.Update, conflictOp) + require.NoError(t, err) + }) + + t.Run("other error should be returned", func(t *testing.T) { + otherError := errors.New("other error") + failOp := func(*badger.Txn) error { + return otherError + } + + err := RetryOnConflict(db.Update, failOp) + require.Equal(t, otherError, err) + }) + }) +} + +func TestSkipNonExists(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("not found", func(t *testing.T) { + op := func(*badger.Txn) error { + return badger.ErrKeyNotFound + } + + err := db.Update(SkipNonExist(op)) + require.NoError(t, err) + }) + + t.Run("not exist", func(t *testing.T) { + op := func(*badger.Txn) error { + return storage.ErrNotFound + } + + err := db.Update(SkipNonExist(op)) + require.NoError(t, err) + }) + + t.Run("general error", func(t *testing.T) { + expectError := fmt.Errorf("random error") + op := func(*badger.Txn) error { + return expectError + } + + err := db.Update(SkipNonExist(op)) + require.Equal(t, expectError, err) + }) + }) +} diff --git a/storage/pebble/operation/prefix.go b/storage/pebble/operation/prefix.go new file mode 100644 index 00000000000..36c33137c80 --- /dev/null +++ b/storage/pebble/operation/prefix.go @@ -0,0 +1,144 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "encoding/binary" + "fmt" + + "github.com/onflow/flow-go/model/flow" +) + +const ( + + // codes for special database markers + codeMax = 1 // keeps track of the maximum key size + codeDBType = 2 // specifies a database type + + // codes for views with special meaning + codeSafetyData = 10 // safety data for hotstuff state + codeLivenessData = 11 // liveness data for hotstuff state + + // codes for fields associated with the root state + codeSporkID = 13 + codeProtocolVersion = 14 + codeEpochCommitSafetyThreshold = 15 + codeSporkRootBlockHeight = 16 + + // code for heights with special meaning + codeFinalizedHeight = 20 // latest finalized block height + codeSealedHeight = 21 // latest sealed block height + codeClusterHeight = 22 // latest finalized height on cluster + codeExecutedBlock = 23 // latest executed block with max height + codeFinalizedRootHeight = 24 // the height of the highest finalized block contained in the root snapshot + codeLastCompleteBlockHeight = 25 // the height of the last block for which all collections were received + codeEpochFirstHeight = 26 // the height of the first block in a given epoch + codeSealedRootHeight = 27 // the height of the highest sealed block contained in the root snapshot + + // codes for single entity storage + // 31 was used for identities before epochs + codeHeader = 30 + codeGuarantee = 32 + codeSeal = 33 + codeTransaction = 34 + codeCollection = 35 + codeExecutionResult = 36 + codeExecutionReceiptMeta = 36 + codeResultApproval = 37 + codeChunk = 38 + + // codes for indexing single identifier by identifier/integeter + codeHeightToBlock = 40 // index mapping height to block ID + codeBlockIDToLatestSealID = 41 // index mapping a block its last payload seal + codeClusterBlockToRefBlock = 42 // index cluster block ID to reference block ID + codeRefHeightToClusterBlock = 43 // index reference block height to cluster block IDs + codeBlockIDToFinalizedSeal = 44 // index _finalized_ seal by sealed block ID + codeBlockIDToQuorumCertificate = 45 // index of quorum certificates by block ID + + // codes for indexing multiple identifiers by identifier + // NOTE: 51 was used for identity indexes before epochs + codeBlockChildren = 50 // index mapping block ID to children blocks + codePayloadGuarantees = 52 // index mapping block ID to payload guarantees + codePayloadSeals = 53 // index mapping block ID to payload seals + codeCollectionBlock = 54 // index mapping collection ID to block ID + codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes + codeBlockEpochStatus = 56 // index mapping block ID to epoch status + codePayloadReceipts = 57 // index mapping block ID to payload receipts + codePayloadResults = 58 // index mapping block ID to payload results + codeAllBlockReceipts = 59 // index mapping of blockID to multiple receipts + + // codes related to protocol level information + codeEpochSetup = 61 // EpochSetup service event, keyed by ID + codeEpochCommit = 62 // EpochCommit service event, keyed by ID + codeBeaconPrivateKey = 63 // BeaconPrivateKey, keyed by epoch counter + codeDKGStarted = 64 // flag that the DKG for an epoch has been started + codeDKGEnded = 65 // flag that the DKG for an epoch has ended (stores end state) + codeVersionBeacon = 67 // flag for storing version beacons + + // code for ComputationResult upload status storage + // NOTE: for now only GCP uploader is supported. When other uploader (AWS e.g.) needs to + // be supported, we will need to define new code. + codeComputationResults = 66 + + // job queue consumers and producers + codeJobConsumerProcessed = 70 + codeJobQueue = 71 + codeJobQueuePointer = 72 + + // legacy codes (should be cleaned up) + codeChunkDataPack = 100 + codeCommit = 101 + codeEvent = 102 + codeExecutionStateInteractions = 103 + codeTransactionResult = 104 + codeFinalizedCluster = 105 + codeServiceEvent = 106 + codeTransactionResultIndex = 107 + codeLightTransactionResult = 108 + codeLightTransactionResultIndex = 109 + codeIndexCollection = 200 + codeIndexExecutionResultByBlock = 202 + codeIndexCollectionByTransaction = 203 + codeIndexResultApprovalByChunk = 204 + + // TEMPORARY codes + blockedNodeIDs = 205 // manual override for adding node IDs to list of ejected nodes, applies to networking layer only + + // internal failure information that should be preserved across restarts + codeExecutionFork = 254 + codeEpochEmergencyFallbackTriggered = 255 +) + +func makePrefix(code byte, keys ...interface{}) []byte { + prefix := make([]byte, 1) + prefix[0] = code + for _, key := range keys { + prefix = append(prefix, b(key)...) + } + return prefix +} + +func b(v interface{}) []byte { + switch i := v.(type) { + case uint8: + return []byte{i} + case uint32: + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, i) + return b + case uint64: + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, i) + return b + case string: + return []byte(i) + case flow.Role: + return []byte{byte(i)} + case flow.Identifier: + return i[:] + case flow.ChainID: + return []byte(i) + default: + panic(fmt.Sprintf("unsupported type to convert (%T)", v)) + } +} diff --git a/storage/pebble/operation/prefix_test.go b/storage/pebble/operation/prefix_test.go new file mode 100644 index 00000000000..4a2af4332e4 --- /dev/null +++ b/storage/pebble/operation/prefix_test.go @@ -0,0 +1,39 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/model/flow" +) + +func TestMakePrefix(t *testing.T) { + + code := byte(0x01) + + u := uint64(1337) + expected := []byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x39} + actual := makePrefix(code, u) + + assert.Equal(t, expected, actual) + + r := flow.Role(2) + expected = []byte{0x01, 0x02} + actual = makePrefix(code, r) + + assert.Equal(t, expected, actual) + + id := flow.Identifier{0x05, 0x06, 0x07} + expected = []byte{0x01, + 0x05, 0x06, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + actual = makePrefix(code, id) + + assert.Equal(t, expected, actual) +} diff --git a/storage/pebble/operation/qcs.go b/storage/pebble/operation/qcs.go new file mode 100644 index 00000000000..651a585b2b2 --- /dev/null +++ b/storage/pebble/operation/qcs.go @@ -0,0 +1,19 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertQuorumCertificate inserts a quorum certificate by block ID. +// Returns storage.ErrAlreadyExists if a QC has already been inserted for the block. +func InsertQuorumCertificate(qc *flow.QuorumCertificate) func(*badger.Txn) error { + return insert(makePrefix(codeBlockIDToQuorumCertificate, qc.BlockID), qc) +} + +// RetrieveQuorumCertificate retrieves a quorum certificate by blockID. +// Returns storage.ErrNotFound if no QC is stored for the block. +func RetrieveQuorumCertificate(blockID flow.Identifier, qc *flow.QuorumCertificate) func(*badger.Txn) error { + return retrieve(makePrefix(codeBlockIDToQuorumCertificate, blockID), qc) +} diff --git a/storage/pebble/operation/qcs_test.go b/storage/pebble/operation/qcs_test.go new file mode 100644 index 00000000000..845f917f041 --- /dev/null +++ b/storage/pebble/operation/qcs_test.go @@ -0,0 +1,27 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertQuorumCertificate(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.QuorumCertificateFixture() + + err := db.Update(InsertQuorumCertificate(expected)) + require.Nil(t, err) + + var actual flow.QuorumCertificate + err = db.View(RetrieveQuorumCertificate(expected.BlockID, &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, &actual) + }) +} diff --git a/storage/pebble/operation/receipts.go b/storage/pebble/operation/receipts.go new file mode 100644 index 00000000000..3dc923af8cb --- /dev/null +++ b/storage/pebble/operation/receipts.go @@ -0,0 +1,87 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertExecutionReceiptMeta inserts an execution receipt meta by ID. +func InsertExecutionReceiptMeta(receiptID flow.Identifier, meta *flow.ExecutionReceiptMeta) func(*badger.Txn) error { + return insert(makePrefix(codeExecutionReceiptMeta, receiptID), meta) +} + +// BatchInsertExecutionReceiptMeta inserts an execution receipt meta by ID. +// TODO: rename to BatchUpdate +func BatchInsertExecutionReceiptMeta(receiptID flow.Identifier, meta *flow.ExecutionReceiptMeta) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeExecutionReceiptMeta, receiptID), meta) +} + +// RetrieveExecutionReceipt retrieves a execution receipt meta by ID. +func RetrieveExecutionReceiptMeta(receiptID flow.Identifier, meta *flow.ExecutionReceiptMeta) func(*badger.Txn) error { + return retrieve(makePrefix(codeExecutionReceiptMeta, receiptID), meta) +} + +// IndexOwnExecutionReceipt inserts an execution receipt ID keyed by block ID +func IndexOwnExecutionReceipt(blockID flow.Identifier, receiptID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeOwnBlockReceipt, blockID), receiptID) +} + +// BatchIndexOwnExecutionReceipt inserts an execution receipt ID keyed by block ID into a batch +// TODO: rename to BatchUpdate +func BatchIndexOwnExecutionReceipt(blockID flow.Identifier, receiptID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeOwnBlockReceipt, blockID), receiptID) +} + +// LookupOwnExecutionReceipt finds execution receipt ID by block +func LookupOwnExecutionReceipt(blockID flow.Identifier, receiptID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeOwnBlockReceipt, blockID), receiptID) +} + +// RemoveOwnExecutionReceipt removes own execution receipt index by blockID +func RemoveOwnExecutionReceipt(blockID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeOwnBlockReceipt, blockID)) +} + +// BatchRemoveOwnExecutionReceipt removes blockID-to-my-receiptID index entries keyed by a blockID in a provided batch. +// No errors are expected during normal operation, but it may return generic error +// if badger fails to process request +func BatchRemoveOwnExecutionReceipt(blockID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchRemove(makePrefix(codeOwnBlockReceipt, blockID)) +} + +// IndexExecutionReceipts inserts an execution receipt ID keyed by block ID and receipt ID. +// one block could have multiple receipts, even if they are from the same executor +func IndexExecutionReceipts(blockID, receiptID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeAllBlockReceipts, blockID, receiptID), receiptID) +} + +// BatchIndexExecutionReceipts inserts an execution receipt ID keyed by block ID and receipt ID into a batch +func BatchIndexExecutionReceipts(blockID, receiptID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeAllBlockReceipts, blockID, receiptID), receiptID) +} + +// LookupExecutionReceipts finds all execution receipts by block ID +func LookupExecutionReceipts(blockID flow.Identifier, receiptIDs *[]flow.Identifier) func(*badger.Txn) error { + iterationFunc := receiptIterationFunc(receiptIDs) + return traverse(makePrefix(codeAllBlockReceipts, blockID), iterationFunc) +} + +// receiptIterationFunc returns an in iteration function which returns all receipt IDs found during traversal +func receiptIterationFunc(receiptIDs *[]flow.Identifier) func() (checkFunc, createFunc, handleFunc) { + check := func(key []byte) bool { + return true + } + + var receiptID flow.Identifier + create := func() interface{} { + return &receiptID + } + handle := func() error { + *receiptIDs = append(*receiptIDs, receiptID) + return nil + } + return func() (checkFunc, createFunc, handleFunc) { + return check, create, handle + } +} diff --git a/storage/pebble/operation/receipts_test.go b/storage/pebble/operation/receipts_test.go new file mode 100644 index 00000000000..1c41f739ebb --- /dev/null +++ b/storage/pebble/operation/receipts_test.go @@ -0,0 +1,64 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestReceipts_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + receipt := unittest.ExecutionReceiptFixture() + expected := receipt.Meta() + + err := db.Update(InsertExecutionReceiptMeta(receipt.ID(), expected)) + require.Nil(t, err) + + var actual flow.ExecutionReceiptMeta + err = db.View(RetrieveExecutionReceiptMeta(receipt.ID(), &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, &actual) + }) +} + +func TestReceipts_Index(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + receipt := unittest.ExecutionReceiptFixture() + expected := receipt.ID() + blockID := receipt.ExecutionResult.BlockID + + err := db.Update(IndexOwnExecutionReceipt(blockID, expected)) + require.Nil(t, err) + + var actual flow.Identifier + err = db.View(LookupOwnExecutionReceipt(blockID, &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, actual) + }) +} + +func TestReceipts_MultiIndex(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := []flow.Identifier{unittest.IdentifierFixture(), unittest.IdentifierFixture()} + blockID := unittest.IdentifierFixture() + + for _, id := range expected { + err := db.Update(IndexExecutionReceipts(blockID, id)) + require.Nil(t, err) + } + var actual []flow.Identifier + err := db.View(LookupExecutionReceipts(blockID, &actual)) + require.Nil(t, err) + + assert.ElementsMatch(t, expected, actual) + }) +} diff --git a/storage/pebble/operation/results.go b/storage/pebble/operation/results.go new file mode 100644 index 00000000000..8e762cc5b41 --- /dev/null +++ b/storage/pebble/operation/results.go @@ -0,0 +1,54 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertExecutionResult inserts an execution result by ID. +func InsertExecutionResult(result *flow.ExecutionResult) func(*badger.Txn) error { + return insert(makePrefix(codeExecutionResult, result.ID()), result) +} + +// BatchInsertExecutionResult inserts an execution result by ID. +func BatchInsertExecutionResult(result *flow.ExecutionResult) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeExecutionResult, result.ID()), result) +} + +// RetrieveExecutionResult retrieves a transaction by fingerprint. +func RetrieveExecutionResult(resultID flow.Identifier, result *flow.ExecutionResult) func(*badger.Txn) error { + return retrieve(makePrefix(codeExecutionResult, resultID), result) +} + +// IndexExecutionResult inserts an execution result ID keyed by block ID +func IndexExecutionResult(blockID flow.Identifier, resultID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeIndexExecutionResultByBlock, blockID), resultID) +} + +// ReindexExecutionResult updates mapping of an execution result ID keyed by block ID +func ReindexExecutionResult(blockID flow.Identifier, resultID flow.Identifier) func(*badger.Txn) error { + return update(makePrefix(codeIndexExecutionResultByBlock, blockID), resultID) +} + +// BatchIndexExecutionResult inserts an execution result ID keyed by block ID into a batch +func BatchIndexExecutionResult(blockID flow.Identifier, resultID flow.Identifier) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeIndexExecutionResultByBlock, blockID), resultID) +} + +// LookupExecutionResult finds execution result ID by block +func LookupExecutionResult(blockID flow.Identifier, resultID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeIndexExecutionResultByBlock, blockID), resultID) +} + +// RemoveExecutionResultIndex removes execution result indexed by the given blockID +func RemoveExecutionResultIndex(blockID flow.Identifier) func(*badger.Txn) error { + return remove(makePrefix(codeIndexExecutionResultByBlock, blockID)) +} + +// BatchRemoveExecutionResultIndex removes blockID-to-resultID index entries keyed by a blockID in a provided batch. +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func BatchRemoveExecutionResultIndex(blockID flow.Identifier) func(*badger.WriteBatch) error { + return batchRemove(makePrefix(codeIndexExecutionResultByBlock, blockID)) +} diff --git a/storage/pebble/operation/results_test.go b/storage/pebble/operation/results_test.go new file mode 100644 index 00000000000..3a3ea267037 --- /dev/null +++ b/storage/pebble/operation/results_test.go @@ -0,0 +1,29 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestResults_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.ExecutionResultFixture() + + err := db.Update(InsertExecutionResult(expected)) + require.Nil(t, err) + + var actual flow.ExecutionResult + err = db.View(RetrieveExecutionResult(expected.ID(), &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, &actual) + }) +} diff --git a/storage/pebble/operation/seals.go b/storage/pebble/operation/seals.go new file mode 100644 index 00000000000..961f9826e34 --- /dev/null +++ b/storage/pebble/operation/seals.go @@ -0,0 +1,77 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func InsertSeal(sealID flow.Identifier, seal *flow.Seal) func(*badger.Txn) error { + return insert(makePrefix(codeSeal, sealID), seal) +} + +func RetrieveSeal(sealID flow.Identifier, seal *flow.Seal) func(*badger.Txn) error { + return retrieve(makePrefix(codeSeal, sealID), seal) +} + +func IndexPayloadSeals(blockID flow.Identifier, sealIDs []flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codePayloadSeals, blockID), sealIDs) +} + +func LookupPayloadSeals(blockID flow.Identifier, sealIDs *[]flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codePayloadSeals, blockID), sealIDs) +} + +func IndexPayloadReceipts(blockID flow.Identifier, receiptIDs []flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codePayloadReceipts, blockID), receiptIDs) +} + +func IndexPayloadResults(blockID flow.Identifier, resultIDs []flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codePayloadResults, blockID), resultIDs) +} + +func LookupPayloadReceipts(blockID flow.Identifier, receiptIDs *[]flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codePayloadReceipts, blockID), receiptIDs) +} + +func LookupPayloadResults(blockID flow.Identifier, resultIDs *[]flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codePayloadResults, blockID), resultIDs) +} + +// IndexLatestSealAtBlock persists the highest seal that was included in the fork up to (and including) blockID. +// In most cases, it is the highest seal included in this block's payload. However, if there are no +// seals in this block, sealID should reference the highest seal in blockID's ancestor. +func IndexLatestSealAtBlock(blockID flow.Identifier, sealID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeBlockIDToLatestSealID, blockID), sealID) +} + +// LookupLatestSealAtBlock finds the highest seal that was included in the fork up to (and including) blockID. +// In most cases, it is the highest seal included in this block's payload. However, if there are no +// seals in this block, sealID should reference the highest seal in blockID's ancestor. +func LookupLatestSealAtBlock(blockID flow.Identifier, sealID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeBlockIDToLatestSealID, blockID), &sealID) +} + +// IndexFinalizedSealByBlockID indexes the _finalized_ seal by the sealed block ID. +// Example: A <- B <- C(SealA) +// when block C is finalized, we create the index `A.ID->SealA.ID` +func IndexFinalizedSealByBlockID(sealedBlockID flow.Identifier, sealID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeBlockIDToFinalizedSeal, sealedBlockID), sealID) +} + +// LookupBySealedBlockID finds the seal for the given sealed block ID. +func LookupBySealedBlockID(sealedBlockID flow.Identifier, sealID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeBlockIDToFinalizedSeal, sealedBlockID), &sealID) +} + +func InsertExecutionForkEvidence(conflictingSeals []*flow.IncorporatedResultSeal) func(*badger.Txn) error { + return insert(makePrefix(codeExecutionFork), conflictingSeals) +} + +func RemoveExecutionForkEvidence() func(*badger.Txn) error { + return remove(makePrefix(codeExecutionFork)) +} + +func RetrieveExecutionForkEvidence(conflictingSeals *[]*flow.IncorporatedResultSeal) func(*badger.Txn) error { + return retrieve(makePrefix(codeExecutionFork), conflictingSeals) +} diff --git a/storage/pebble/operation/seals_test.go b/storage/pebble/operation/seals_test.go new file mode 100644 index 00000000000..73846bbfbed --- /dev/null +++ b/storage/pebble/operation/seals_test.go @@ -0,0 +1,61 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestSealInsertCheckRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.Seal.Fixture() + + err := db.Update(InsertSeal(expected.ID(), expected)) + require.Nil(t, err) + + var actual flow.Seal + err = db.View(RetrieveSeal(expected.ID(), &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, &actual) + }) +} + +func TestSealIndexAndLookup(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + seal1 := unittest.Seal.Fixture() + seal2 := unittest.Seal.Fixture() + + seals := []*flow.Seal{seal1, seal2} + + blockID := flow.MakeID([]byte{0x42}) + + expected := []flow.Identifier(flow.GetIDs(seals)) + + err := db.Update(func(tx *badger.Txn) error { + for _, seal := range seals { + if err := InsertSeal(seal.ID(), seal)(tx); err != nil { + return err + } + } + if err := IndexPayloadSeals(blockID, expected)(tx); err != nil { + return err + } + return nil + }) + require.Nil(t, err) + + var actual []flow.Identifier + err = db.View(LookupPayloadSeals(blockID, &actual)) + require.Nil(t, err) + + assert.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/operation/spork.go b/storage/pebble/operation/spork.go new file mode 100644 index 00000000000..9f80afcddf9 --- /dev/null +++ b/storage/pebble/operation/spork.go @@ -0,0 +1,59 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertSporkID inserts the spork ID for the present spork. A single database +// and protocol state instance spans at most one spork, so this is inserted +// exactly once, when bootstrapping the state. +func InsertSporkID(sporkID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codeSporkID), sporkID) +} + +// RetrieveSporkID retrieves the spork ID for the present spork. +func RetrieveSporkID(sporkID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codeSporkID), sporkID) +} + +// InsertSporkRootBlockHeight inserts the spork root block height for the present spork. +// A single database and protocol state instance spans at most one spork, so this is inserted +// exactly once, when bootstrapping the state. +func InsertSporkRootBlockHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeSporkRootBlockHeight), height) +} + +// RetrieveSporkRootBlockHeight retrieves the spork root block height for the present spork. +func RetrieveSporkRootBlockHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeSporkRootBlockHeight), height) +} + +// InsertProtocolVersion inserts the protocol version for the present spork. +// A single database and protocol state instance spans at most one spork, and +// a spork has exactly one protocol version for its duration, so this is +// inserted exactly once, when bootstrapping the state. +func InsertProtocolVersion(version uint) func(*badger.Txn) error { + return insert(makePrefix(codeProtocolVersion), version) +} + +// RetrieveProtocolVersion retrieves the protocol version for the present spork. +func RetrieveProtocolVersion(version *uint) func(*badger.Txn) error { + return retrieve(makePrefix(codeProtocolVersion), version) +} + +// InsertEpochCommitSafetyThreshold inserts the epoch commit safety threshold +// for the present spork. +// A single database and protocol state instance spans at most one spork, and +// a spork has exactly one protocol version for its duration, so this is +// inserted exactly once, when bootstrapping the state. +func InsertEpochCommitSafetyThreshold(threshold uint64) func(*badger.Txn) error { + return insert(makePrefix(codeEpochCommitSafetyThreshold), threshold) +} + +// RetrieveEpochCommitSafetyThreshold retrieves the epoch commit safety threshold +// for the present spork. +func RetrieveEpochCommitSafetyThreshold(threshold *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochCommitSafetyThreshold), threshold) +} diff --git a/storage/pebble/operation/spork_test.go b/storage/pebble/operation/spork_test.go new file mode 100644 index 00000000000..a000df60561 --- /dev/null +++ b/storage/pebble/operation/spork_test.go @@ -0,0 +1,60 @@ +package operation + +import ( + "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestSporkID_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + sporkID := unittest.IdentifierFixture() + + err := db.Update(InsertSporkID(sporkID)) + require.NoError(t, err) + + var actual flow.Identifier + err = db.View(RetrieveSporkID(&actual)) + require.NoError(t, err) + + assert.Equal(t, sporkID, actual) + }) +} + +func TestProtocolVersion_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + version := uint(rand.Uint32()) + + err := db.Update(InsertProtocolVersion(version)) + require.NoError(t, err) + + var actual uint + err = db.View(RetrieveProtocolVersion(&actual)) + require.NoError(t, err) + + assert.Equal(t, version, actual) + }) +} + +// TestEpochCommitSafetyThreshold_InsertRetrieve tests that we can insert and +// retrieve epoch commit safety threshold values. +func TestEpochCommitSafetyThreshold_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + threshold := rand.Uint64() + + err := db.Update(InsertEpochCommitSafetyThreshold(threshold)) + require.NoError(t, err) + + var actual uint64 + err = db.View(RetrieveEpochCommitSafetyThreshold(&actual)) + require.NoError(t, err) + + assert.Equal(t, threshold, actual) + }) +} diff --git a/storage/pebble/operation/transaction_results.go b/storage/pebble/operation/transaction_results.go new file mode 100644 index 00000000000..ed215aaedf7 --- /dev/null +++ b/storage/pebble/operation/transaction_results.go @@ -0,0 +1,124 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package operation + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +func InsertTransactionResult(blockID flow.Identifier, transactionResult *flow.TransactionResult) func(*badger.Txn) error { + return insert(makePrefix(codeTransactionResult, blockID, transactionResult.TransactionID), transactionResult) +} + +func BatchInsertTransactionResult(blockID flow.Identifier, transactionResult *flow.TransactionResult) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeTransactionResult, blockID, transactionResult.TransactionID), transactionResult) +} + +func BatchIndexTransactionResult(blockID flow.Identifier, txIndex uint32, transactionResult *flow.TransactionResult) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeTransactionResultIndex, blockID, txIndex), transactionResult) +} + +func RetrieveTransactionResult(blockID flow.Identifier, transactionID flow.Identifier, transactionResult *flow.TransactionResult) func(*badger.Txn) error { + return retrieve(makePrefix(codeTransactionResult, blockID, transactionID), transactionResult) +} +func RetrieveTransactionResultByIndex(blockID flow.Identifier, txIndex uint32, transactionResult *flow.TransactionResult) func(*badger.Txn) error { + return retrieve(makePrefix(codeTransactionResultIndex, blockID, txIndex), transactionResult) +} + +// LookupTransactionResultsByBlockIDUsingIndex retrieves all tx results for a block, by using +// tx_index index. This correctly handles cases of duplicate transactions within block. +func LookupTransactionResultsByBlockIDUsingIndex(blockID flow.Identifier, txResults *[]flow.TransactionResult) func(*badger.Txn) error { + + txErrIterFunc := func() (checkFunc, createFunc, handleFunc) { + check := func(_ []byte) bool { + return true + } + var val flow.TransactionResult + create := func() interface{} { + return &val + } + handle := func() error { + *txResults = append(*txResults, val) + return nil + } + return check, create, handle + } + + return traverse(makePrefix(codeTransactionResultIndex, blockID), txErrIterFunc) +} + +// RemoveTransactionResultsByBlockID removes the transaction results for the given blockID +func RemoveTransactionResultsByBlockID(blockID flow.Identifier) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + + prefix := makePrefix(codeTransactionResult, blockID) + err := removeByPrefix(prefix)(txn) + if err != nil { + return fmt.Errorf("could not remove transaction results for block %v: %w", blockID, err) + } + + return nil + } +} + +// BatchRemoveTransactionResultsByBlockID removes transaction results for the given blockID in a provided batch. +// No errors are expected during normal operation, but it may return generic error +// if badger fails to process request +func BatchRemoveTransactionResultsByBlockID(blockID flow.Identifier, batch *badger.WriteBatch) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + + prefix := makePrefix(codeTransactionResult, blockID) + err := batchRemoveByPrefix(prefix)(txn, batch) + if err != nil { + return fmt.Errorf("could not remove transaction results for block %v: %w", blockID, err) + } + + return nil + } +} + +func InsertLightTransactionResult(blockID flow.Identifier, transactionResult *flow.LightTransactionResult) func(*badger.Txn) error { + return insert(makePrefix(codeLightTransactionResult, blockID, transactionResult.TransactionID), transactionResult) +} + +func BatchInsertLightTransactionResult(blockID flow.Identifier, transactionResult *flow.LightTransactionResult) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeLightTransactionResult, blockID, transactionResult.TransactionID), transactionResult) +} + +func BatchIndexLightTransactionResult(blockID flow.Identifier, txIndex uint32, transactionResult *flow.LightTransactionResult) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeLightTransactionResultIndex, blockID, txIndex), transactionResult) +} + +func RetrieveLightTransactionResult(blockID flow.Identifier, transactionID flow.Identifier, transactionResult *flow.LightTransactionResult) func(*badger.Txn) error { + return retrieve(makePrefix(codeLightTransactionResult, blockID, transactionID), transactionResult) +} + +func RetrieveLightTransactionResultByIndex(blockID flow.Identifier, txIndex uint32, transactionResult *flow.LightTransactionResult) func(*badger.Txn) error { + return retrieve(makePrefix(codeLightTransactionResultIndex, blockID, txIndex), transactionResult) +} + +// LookupLightTransactionResultsByBlockIDUsingIndex retrieves all tx results for a block, but using +// tx_index index. This correctly handles cases of duplicate transactions within block. +func LookupLightTransactionResultsByBlockIDUsingIndex(blockID flow.Identifier, txResults *[]flow.LightTransactionResult) func(*badger.Txn) error { + + txErrIterFunc := func() (checkFunc, createFunc, handleFunc) { + check := func(_ []byte) bool { + return true + } + var val flow.LightTransactionResult + create := func() interface{} { + return &val + } + handle := func() error { + *txResults = append(*txResults, val) + return nil + } + return check, create, handle + } + + return traverse(makePrefix(codeLightTransactionResultIndex, blockID), txErrIterFunc) +} diff --git a/storage/pebble/operation/transactions.go b/storage/pebble/operation/transactions.go new file mode 100644 index 00000000000..1ad372bc6a7 --- /dev/null +++ b/storage/pebble/operation/transactions.go @@ -0,0 +1,17 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// InsertTransaction inserts a transaction keyed by transaction fingerprint. +func InsertTransaction(txID flow.Identifier, tx *flow.TransactionBody) func(*badger.Txn) error { + return insert(makePrefix(codeTransaction, txID), tx) +} + +// RetrieveTransaction retrieves a transaction by fingerprint. +func RetrieveTransaction(txID flow.Identifier, tx *flow.TransactionBody) func(*badger.Txn) error { + return retrieve(makePrefix(codeTransaction, txID), tx) +} diff --git a/storage/pebble/operation/transactions_test.go b/storage/pebble/operation/transactions_test.go new file mode 100644 index 00000000000..f3b34f7d0ff --- /dev/null +++ b/storage/pebble/operation/transactions_test.go @@ -0,0 +1,26 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestTransactions(t *testing.T) { + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + expected := unittest.TransactionFixture() + err := db.Update(InsertTransaction(expected.ID(), &expected.TransactionBody)) + require.Nil(t, err) + + var actual flow.Transaction + err = db.View(RetrieveTransaction(expected.ID(), &actual.TransactionBody)) + require.Nil(t, err) + assert.Equal(t, expected, actual) + }) +} diff --git a/storage/pebble/operation/version_beacon.go b/storage/pebble/operation/version_beacon.go new file mode 100644 index 00000000000..a90ae58e4fb --- /dev/null +++ b/storage/pebble/operation/version_beacon.go @@ -0,0 +1,31 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// IndexVersionBeaconByHeight stores a sealed version beacon indexed by +// flow.SealedVersionBeacon.SealHeight. +// +// No errors are expected during normal operation. +func IndexVersionBeaconByHeight( + beacon *flow.SealedVersionBeacon, +) func(*badger.Txn) error { + return upsert(makePrefix(codeVersionBeacon, beacon.SealHeight), beacon) +} + +// LookupLastVersionBeaconByHeight finds the highest flow.VersionBeacon but no higher +// than maxHeight. Returns storage.ErrNotFound if no version beacon exists at or below +// the given height. +func LookupLastVersionBeaconByHeight( + maxHeight uint64, + versionBeacon *flow.SealedVersionBeacon, +) func(*badger.Txn) error { + return findHighestAtOrBelow( + makePrefix(codeVersionBeacon), + maxHeight, + versionBeacon, + ) +} diff --git a/storage/pebble/operation/version_beacon_test.go b/storage/pebble/operation/version_beacon_test.go new file mode 100644 index 00000000000..d46ed334f93 --- /dev/null +++ b/storage/pebble/operation/version_beacon_test.go @@ -0,0 +1,106 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestResults_IndexByServiceEvents(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height1 := uint64(21) + height2 := uint64(37) + height3 := uint64(55) + vb1 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "1.0.0", + BlockHeight: height1 + 5, + }, + ), + ), + SealHeight: height1, + } + vb2 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "1.1.0", + BlockHeight: height2 + 5, + }, + ), + ), + SealHeight: height2, + } + vb3 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "2.0.0", + BlockHeight: height3 + 5, + }, + ), + ), + SealHeight: height3, + } + + // indexing 3 version beacons at different heights + err := db.Update(IndexVersionBeaconByHeight(&vb1)) + require.NoError(t, err) + + err = db.Update(IndexVersionBeaconByHeight(&vb2)) + require.NoError(t, err) + + err = db.Update(IndexVersionBeaconByHeight(&vb3)) + require.NoError(t, err) + + // index version beacon 2 again to make sure we tolerate duplicates + // it is possible for two or more events of the same type to be from the same height + err = db.Update(IndexVersionBeaconByHeight(&vb2)) + require.NoError(t, err) + + t.Run("retrieve exact height match", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + err := db.View(LookupLastVersionBeaconByHeight(height1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb1, actualVB) + + err = db.View(LookupLastVersionBeaconByHeight(height2, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb2, actualVB) + + err = db.View(LookupLastVersionBeaconByHeight(height3, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb3, actualVB) + }) + + t.Run("finds highest but not higher than given", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height3-1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb2, actualVB) + }) + + t.Run("finds highest", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height3+1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb3, actualVB) + }) + + t.Run("height below lowest entry returns nothing", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height1-1, &actualVB)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) + }) +} diff --git a/storage/pebble/operation/views.go b/storage/pebble/operation/views.go new file mode 100644 index 00000000000..21f31316f1f --- /dev/null +++ b/storage/pebble/operation/views.go @@ -0,0 +1,38 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" +) + +// InsertSafetyData inserts safety data into the database. +func InsertSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return insert(makePrefix(codeSafetyData, chainID), safetyData) +} + +// UpdateSafetyData updates safety data in the database. +func UpdateSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return update(makePrefix(codeSafetyData, chainID), safetyData) +} + +// RetrieveSafetyData retrieves safety data from the database. +func RetrieveSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return retrieve(makePrefix(codeSafetyData, chainID), safetyData) +} + +// InsertLivenessData inserts liveness data into the database. +func InsertLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return insert(makePrefix(codeLivenessData, chainID), livenessData) +} + +// UpdateLivenessData updates liveness data in the database. +func UpdateLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return update(makePrefix(codeLivenessData, chainID), livenessData) +} + +// RetrieveLivenessData retrieves liveness data from the database. +func RetrieveLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return retrieve(makePrefix(codeLivenessData, chainID), livenessData) +} diff --git a/storage/pebble/payloads.go b/storage/pebble/payloads.go new file mode 100644 index 00000000000..ec75103cde3 --- /dev/null +++ b/storage/pebble/payloads.go @@ -0,0 +1,165 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type Payloads struct { + db *badger.DB + index *Index + guarantees *Guarantees + seals *Seals + receipts *ExecutionReceipts + results *ExecutionResults +} + +func NewPayloads(db *badger.DB, index *Index, guarantees *Guarantees, seals *Seals, receipts *ExecutionReceipts, + results *ExecutionResults) *Payloads { + + p := &Payloads{ + db: db, + index: index, + guarantees: guarantees, + seals: seals, + receipts: receipts, + results: results, + } + + return p +} + +func (p *Payloads) storeTx(blockID flow.Identifier, payload *flow.Payload) func(*transaction.Tx) error { + // For correct payloads, the execution result is part of the payload or it's already stored + // in storage. If execution result is not present in either of those places, we error. + // ATTENTION: this is unnecessarily complex if we have execution receipt which points an execution result + // which is not included in current payload but was incorporated in one of previous blocks. + + return func(tx *transaction.Tx) error { + + resultsByID := payload.Results.Lookup() + fullReceipts := make([]*flow.ExecutionReceipt, 0, len(payload.Receipts)) + var err error + for _, meta := range payload.Receipts { + result, ok := resultsByID[meta.ResultID] + if !ok { + result, err = p.results.ByIDTx(meta.ResultID)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + err = fmt.Errorf("invalid payload referencing unknown execution result %v, err: %w", meta.ResultID, err) + } + return err + } + } + fullReceipts = append(fullReceipts, flow.ExecutionReceiptFromMeta(*meta, *result)) + } + + // make sure all payload guarantees are stored + for _, guarantee := range payload.Guarantees { + err := p.guarantees.storeTx(guarantee)(tx) + if err != nil { + return fmt.Errorf("could not store guarantee: %w", err) + } + } + + // make sure all payload seals are stored + for _, seal := range payload.Seals { + err := p.seals.storeTx(seal)(tx) + if err != nil { + return fmt.Errorf("could not store seal: %w", err) + } + } + + // store all payload receipts + for _, receipt := range fullReceipts { + err := p.receipts.storeTx(receipt)(tx) + if err != nil { + return fmt.Errorf("could not store receipt: %w", err) + } + } + + // store the index + err = p.index.storeTx(blockID, payload.Index())(tx) + if err != nil { + return fmt.Errorf("could not store index: %w", err) + } + + return nil + } +} + +func (p *Payloads) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn) (*flow.Payload, error) { + return func(tx *badger.Txn) (*flow.Payload, error) { + + // retrieve the index + idx, err := p.index.retrieveTx(blockID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve index: %w", err) + } + + // retrieve guarantees + guarantees := make([]*flow.CollectionGuarantee, 0, len(idx.CollectionIDs)) + for _, collID := range idx.CollectionIDs { + guarantee, err := p.guarantees.retrieveTx(collID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve guarantee (%x): %w", collID, err) + } + guarantees = append(guarantees, guarantee) + } + + // retrieve seals + seals := make([]*flow.Seal, 0, len(idx.SealIDs)) + for _, sealID := range idx.SealIDs { + seal, err := p.seals.retrieveTx(sealID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve seal (%x): %w", sealID, err) + } + seals = append(seals, seal) + } + + // retrieve receipts + receipts := make([]*flow.ExecutionReceiptMeta, 0, len(idx.ReceiptIDs)) + for _, recID := range idx.ReceiptIDs { + receipt, err := p.receipts.byID(recID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve receipt %x: %w", recID, err) + } + receipts = append(receipts, receipt.Meta()) + } + + // retrieve results + results := make([]*flow.ExecutionResult, 0, len(idx.ResultIDs)) + for _, resID := range idx.ResultIDs { + result, err := p.results.byID(resID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve result %x: %w", resID, err) + } + results = append(results, result) + } + payload := &flow.Payload{ + Seals: seals, + Guarantees: guarantees, + Receipts: receipts, + Results: results, + } + + return payload, nil + } +} + +func (p *Payloads) Store(blockID flow.Identifier, payload *flow.Payload) error { + return operation.RetryOnConflictTx(p.db, transaction.Update, p.storeTx(blockID, payload)) +} + +func (p *Payloads) ByBlockID(blockID flow.Identifier) (*flow.Payload, error) { + tx := p.db.NewTransaction(false) + defer tx.Discard() + return p.retrieveTx(blockID)(tx) +} diff --git a/storage/pebble/payloads_test.go b/storage/pebble/payloads_test.go new file mode 100644 index 00000000000..cb11074f88b --- /dev/null +++ b/storage/pebble/payloads_test.go @@ -0,0 +1,59 @@ +package badger_test + +import ( + "errors" + + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestPayloadStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + + index := badgerstorage.NewIndex(metrics, db) + seals := badgerstorage.NewSeals(metrics, db) + guarantees := badgerstorage.NewGuarantees(metrics, db, badgerstorage.DefaultCacheSize) + results := badgerstorage.NewExecutionResults(metrics, db) + receipts := badgerstorage.NewExecutionReceipts(metrics, db, results, badgerstorage.DefaultCacheSize) + store := badgerstorage.NewPayloads(db, index, guarantees, seals, receipts, results) + + blockID := unittest.IdentifierFixture() + expected := unittest.PayloadFixture(unittest.WithAllTheFixins) + + // store payload + err := store.Store(blockID, &expected) + require.NoError(t, err) + + // fetch payload + payload, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Equal(t, &expected, payload) + }) +} + +func TestPayloadRetreiveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + + index := badgerstorage.NewIndex(metrics, db) + seals := badgerstorage.NewSeals(metrics, db) + guarantees := badgerstorage.NewGuarantees(metrics, db, badgerstorage.DefaultCacheSize) + results := badgerstorage.NewExecutionResults(metrics, db) + receipts := badgerstorage.NewExecutionReceipts(metrics, db, results, badgerstorage.DefaultCacheSize) + store := badgerstorage.NewPayloads(db, index, guarantees, seals, receipts, results) + + blockID := unittest.IdentifierFixture() + + _, err := store.ByBlockID(blockID) + require.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} diff --git a/storage/pebble/procedure/children.go b/storage/pebble/procedure/children.go new file mode 100644 index 00000000000..e95412f6403 --- /dev/null +++ b/storage/pebble/procedure/children.go @@ -0,0 +1,82 @@ +package procedure + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// IndexNewBlock will add parent-child index for the new block. +// - Each block has a parent, we use this parent-child relationship to build a reverse index +// - for looking up children blocks for a given block. This is useful for forks recovery +// where we want to find all the pending children blocks for the lastest finalized block. +// +// When adding parent-child index for a new block, we will add two indexes: +// 1. since it's a new block, the new block should have no child, so adding an empty +// index for the new block. Note: It's impossible there is a block whose parent is the +// new block. +// 2. since the parent block has this new block as a child, adding an index for that. +// there are two special cases for (2): +// - if the parent block is zero, then we don't need to add this index. +// - if the parent block doesn't exist, then we will insert the child index instead of updating +func IndexNewBlock(blockID flow.Identifier, parentID flow.Identifier) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + // Step 1: index the child for the new block. + // the new block has no child, so adding an empty child index for it + err := operation.InsertBlockChildren(blockID, nil)(tx) + if err != nil { + return fmt.Errorf("could not insert empty block children: %w", err) + } + + // Step 2: adding the second index for the parent block + // if the parent block is zero, for instance root block has no parent, + // then no need to add index for it + if parentID == flow.ZeroID { + return nil + } + + // if the parent block is not zero, depending on whether the parent block has + // children or not, we will either update the index or insert the index: + // when parent block doesn't exist, we will insert the block children. + // when parent block exists already, we will update the block children, + var childrenIDs flow.IdentifierList + err = operation.RetrieveBlockChildren(parentID, &childrenIDs)(tx) + + var saveIndex func(blockID flow.Identifier, childrenIDs flow.IdentifierList) func(*badger.Txn) error + if errors.Is(err, storage.ErrNotFound) { + saveIndex = operation.InsertBlockChildren + } else if err != nil { + return fmt.Errorf("could not look up block children: %w", err) + } else { // err == nil + saveIndex = operation.UpdateBlockChildren + } + + // check we don't add a duplicate + for _, dupID := range childrenIDs { + if blockID == dupID { + return storage.ErrAlreadyExists + } + } + + // adding the new block to be another child of the parent + childrenIDs = append(childrenIDs, blockID) + + // saving the index + err = saveIndex(parentID, childrenIDs)(tx) + if err != nil { + return fmt.Errorf("could not update children index: %w", err) + } + + return nil + } +} + +// LookupBlockChildren looks up the IDs of all child blocks of the given parent block. +func LookupBlockChildren(blockID flow.Identifier, childrenIDs *flow.IdentifierList) func(tx *badger.Txn) error { + return operation.RetrieveBlockChildren(blockID, childrenIDs) +} diff --git a/storage/pebble/procedure/children_test.go b/storage/pebble/procedure/children_test.go new file mode 100644 index 00000000000..9cf6a71773f --- /dev/null +++ b/storage/pebble/procedure/children_test.go @@ -0,0 +1,115 @@ +package procedure_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/unittest" +) + +// after indexing a block by its parent, it should be able to retrieve the child block by the parentID +func TestIndexAndLookupChild(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + parentID := unittest.IdentifierFixture() + childID := unittest.IdentifierFixture() + + err := db.Update(procedure.IndexNewBlock(childID, parentID)) + require.NoError(t, err) + + // retrieve child + var retrievedIDs flow.IdentifierList + err = db.View(procedure.LookupBlockChildren(parentID, &retrievedIDs)) + require.NoError(t, err) + + // retrieved child should be the stored child + require.Equal(t, flow.IdentifierList{childID}, retrievedIDs) + }) +} + +// if two blocks connect to the same parent, indexing the second block would have +// no effect, retrieving the child of the parent block will return the first block that +// was indexed. +func TestIndexTwiceAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + parentID := unittest.IdentifierFixture() + child1ID := unittest.IdentifierFixture() + child2ID := unittest.IdentifierFixture() + + // index the first child + err := db.Update(procedure.IndexNewBlock(child1ID, parentID)) + require.NoError(t, err) + + // index the second child + err = db.Update(procedure.IndexNewBlock(child2ID, parentID)) + require.NoError(t, err) + + var retrievedIDs flow.IdentifierList + err = db.View(procedure.LookupBlockChildren(parentID, &retrievedIDs)) + require.NoError(t, err) + + require.Equal(t, flow.IdentifierList{child1ID, child2ID}, retrievedIDs) + }) +} + +// if parent is zero, then we don't index it +func TestIndexZeroParent(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + childID := unittest.IdentifierFixture() + + err := db.Update(procedure.IndexNewBlock(childID, flow.ZeroID)) + require.NoError(t, err) + + // zero id should have no children + var retrievedIDs flow.IdentifierList + err = db.View(procedure.LookupBlockChildren(flow.ZeroID, &retrievedIDs)) + require.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} + +// lookup block children will only return direct childrens +func TestDirectChildren(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + + b1 := unittest.IdentifierFixture() + b2 := unittest.IdentifierFixture() + b3 := unittest.IdentifierFixture() + b4 := unittest.IdentifierFixture() + + err := db.Update(procedure.IndexNewBlock(b2, b1)) + require.NoError(t, err) + + err = db.Update(procedure.IndexNewBlock(b3, b2)) + require.NoError(t, err) + + err = db.Update(procedure.IndexNewBlock(b4, b3)) + require.NoError(t, err) + + // check the children of the first block + var retrievedIDs flow.IdentifierList + + err = db.View(procedure.LookupBlockChildren(b1, &retrievedIDs)) + require.NoError(t, err) + require.Equal(t, flow.IdentifierList{b2}, retrievedIDs) + + err = db.View(procedure.LookupBlockChildren(b2, &retrievedIDs)) + require.NoError(t, err) + require.Equal(t, flow.IdentifierList{b3}, retrievedIDs) + + err = db.View(procedure.LookupBlockChildren(b3, &retrievedIDs)) + require.NoError(t, err) + require.Equal(t, flow.IdentifierList{b4}, retrievedIDs) + + err = db.View(procedure.LookupBlockChildren(b4, &retrievedIDs)) + require.NoError(t, err) + require.Nil(t, retrievedIDs) + }) +} diff --git a/storage/pebble/procedure/cluster.go b/storage/pebble/procedure/cluster.go new file mode 100644 index 00000000000..f51c8597938 --- /dev/null +++ b/storage/pebble/procedure/cluster.go @@ -0,0 +1,225 @@ +package procedure + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// This file implements storage functions for blocks in cluster consensus. + +// InsertClusterBlock inserts a cluster consensus block, updating all +// associated indexes. +func InsertClusterBlock(block *cluster.Block) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // check payload integrity + if block.Header.PayloadHash != block.Payload.Hash() { + return fmt.Errorf("computed payload hash does not match header") + } + + // store the block header + blockID := block.ID() + err := operation.InsertHeader(blockID, block.Header)(tx) + if err != nil { + return fmt.Errorf("could not insert header: %w", err) + } + + // insert the block payload + err = InsertClusterPayload(blockID, block.Payload)(tx) + if err != nil { + return fmt.Errorf("could not insert payload: %w", err) + } + + // index the child block for recovery + err = IndexNewBlock(blockID, block.Header.ParentID)(tx) + if err != nil { + return fmt.Errorf("could not index new block: %w", err) + } + return nil + } +} + +// RetrieveClusterBlock retrieves a cluster consensus block by block ID. +func RetrieveClusterBlock(blockID flow.Identifier, block *cluster.Block) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // retrieve the block header + var header flow.Header + err := operation.RetrieveHeader(blockID, &header)(tx) + if err != nil { + return fmt.Errorf("could not retrieve header: %w", err) + } + + // retrieve payload + var payload cluster.Payload + err = RetrieveClusterPayload(blockID, &payload)(tx) + if err != nil { + return fmt.Errorf("could not retrieve payload: %w", err) + } + + // overwrite block + *block = cluster.Block{ + Header: &header, + Payload: &payload, + } + + return nil + } +} + +// RetrieveLatestFinalizedClusterHeader retrieves the latest finalized for the +// given cluster chain ID. +func RetrieveLatestFinalizedClusterHeader(chainID flow.ChainID, final *flow.Header) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + var boundary uint64 + err := operation.RetrieveClusterFinalizedHeight(chainID, &boundary)(tx) + if err != nil { + return fmt.Errorf("could not retrieve boundary: %w", err) + } + + var finalID flow.Identifier + err = operation.LookupClusterBlockHeight(chainID, boundary, &finalID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve final ID: %w", err) + } + + err = operation.RetrieveHeader(finalID, final)(tx) + if err != nil { + return fmt.Errorf("could not retrieve finalized header: %w", err) + } + + return nil + } +} + +// FinalizeClusterBlock finalizes a block in cluster consensus. +func FinalizeClusterBlock(blockID flow.Identifier) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // retrieve the header to check the parent + var header flow.Header + err := operation.RetrieveHeader(blockID, &header)(tx) + if err != nil { + return fmt.Errorf("could not retrieve header: %w", err) + } + + // get the chain ID, which determines which cluster state to query + chainID := header.ChainID + + // retrieve the current finalized state boundary + var boundary uint64 + err = operation.RetrieveClusterFinalizedHeight(chainID, &boundary)(tx) + if err != nil { + return fmt.Errorf("could not retrieve boundary: %w", err) + } + + // retrieve the ID of the boundary head + var headID flow.Identifier + err = operation.LookupClusterBlockHeight(chainID, boundary, &headID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve head: %w", err) + } + + // check that the head ID is the parent of the block we finalize + if header.ParentID != headID { + return fmt.Errorf("can't finalize non-child of chain head") + } + + // insert block view -> ID mapping + err = operation.IndexClusterBlockHeight(chainID, header.Height, header.ID())(tx) + if err != nil { + return fmt.Errorf("could not insert view->ID mapping: %w", err) + } + + // update the finalized boundary + err = operation.UpdateClusterFinalizedHeight(chainID, header.Height)(tx) + if err != nil { + return fmt.Errorf("could not update finalized boundary: %w", err) + } + + // NOTE: we don't want to prune forks that have become invalid here, so + // that we can keep validating entities and generating slashing + // challenges for some time - the pruning should happen some place else + // after a certain delay of blocks + + return nil + } +} + +// InsertClusterPayload inserts the payload for a cluster block. It inserts +// both the collection and all constituent transactions, allowing duplicates. +func InsertClusterPayload(blockID flow.Identifier, payload *cluster.Payload) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // cluster payloads only contain a single collection, allow duplicates, + // because it is valid for two competing forks to have the same payload. + light := payload.Collection.Light() + err := operation.SkipDuplicates(operation.InsertCollection(&light))(tx) + if err != nil { + return fmt.Errorf("could not insert payload collection: %w", err) + } + + // insert constituent transactions + for _, colTx := range payload.Collection.Transactions { + err = operation.SkipDuplicates(operation.InsertTransaction(colTx.ID(), colTx))(tx) + if err != nil { + return fmt.Errorf("could not insert payload transaction: %w", err) + } + } + + // index the transaction IDs within the collection + txIDs := payload.Collection.Light().Transactions + err = operation.SkipDuplicates(operation.IndexCollectionPayload(blockID, txIDs))(tx) + if err != nil { + return fmt.Errorf("could not index collection: %w", err) + } + + // insert the reference block ID + err = operation.IndexReferenceBlockByClusterBlock(blockID, payload.ReferenceBlockID)(tx) + if err != nil { + return fmt.Errorf("could not insert reference block ID: %w", err) + } + + return nil + } +} + +// RetrieveClusterPayload retrieves a cluster consensus block payload by block ID. +func RetrieveClusterPayload(blockID flow.Identifier, payload *cluster.Payload) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + + // lookup the reference block ID + var refID flow.Identifier + err := operation.LookupReferenceBlockByClusterBlock(blockID, &refID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve reference block ID: %w", err) + } + + // lookup collection transaction IDs + var txIDs []flow.Identifier + err = operation.LookupCollectionPayload(blockID, &txIDs)(tx) + if err != nil { + return fmt.Errorf("could not look up collection payload: %w", err) + } + + colTransactions := make([]*flow.TransactionBody, 0, len(txIDs)) + // retrieve individual transactions + for _, txID := range txIDs { + var nextTx flow.TransactionBody + err = operation.RetrieveTransaction(txID, &nextTx)(tx) + if err != nil { + return fmt.Errorf("could not retrieve transaction: %w", err) + } + colTransactions = append(colTransactions, &nextTx) + } + + *payload = cluster.PayloadFromTransactions(refID, colTransactions...) + + return nil + } +} diff --git a/storage/pebble/procedure/cluster_test.go b/storage/pebble/procedure/cluster_test.go new file mode 100644 index 00000000000..325c7919454 --- /dev/null +++ b/storage/pebble/procedure/cluster_test.go @@ -0,0 +1,58 @@ +package procedure + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertRetrieveClusterBlock(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + block := unittest.ClusterBlockFixture() + + err := db.Update(InsertClusterBlock(&block)) + require.NoError(t, err) + + var retrieved cluster.Block + err = db.View(RetrieveClusterBlock(block.Header.ID(), &retrieved)) + require.NoError(t, err) + + require.Equal(t, block, retrieved) + }) +} + +func TestFinalizeClusterBlock(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + parent := unittest.ClusterBlockFixture() + + block := unittest.ClusterBlockWithParent(&parent) + + err := db.Update(InsertClusterBlock(&block)) + require.NoError(t, err) + + err = db.Update(operation.IndexClusterBlockHeight(block.Header.ChainID, parent.Header.Height, parent.ID())) + require.NoError(t, err) + + err = db.Update(operation.InsertClusterFinalizedHeight(block.Header.ChainID, parent.Header.Height)) + require.NoError(t, err) + + err = db.Update(FinalizeClusterBlock(block.Header.ID())) + require.NoError(t, err) + + var boundary uint64 + err = db.View(operation.RetrieveClusterFinalizedHeight(block.Header.ChainID, &boundary)) + require.NoError(t, err) + require.Equal(t, block.Header.Height, boundary) + + var headID flow.Identifier + err = db.View(operation.LookupClusterBlockHeight(block.Header.ChainID, boundary, &headID)) + require.NoError(t, err) + require.Equal(t, block.ID(), headID) + }) +} diff --git a/storage/pebble/procedure/executed.go b/storage/pebble/procedure/executed.go new file mode 100644 index 00000000000..eb6a094f638 --- /dev/null +++ b/storage/pebble/procedure/executed.go @@ -0,0 +1,63 @@ +package procedure + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +// UpdateHighestExecutedBlockIfHigher updates the latest executed block to be the input block +// if the input block has a greater height than the currently stored latest executed block. +// The executed block index must have been initialized before calling this function. +// Returns storage.ErrNotFound if the input block does not exist in storage. +func UpdateHighestExecutedBlockIfHigher(header *flow.Header) func(txn *badger.Txn) error { + return func(txn *badger.Txn) error { + var blockID flow.Identifier + err := operation.RetrieveExecutedBlock(&blockID)(txn) + if err != nil { + return fmt.Errorf("cannot lookup executed block: %w", err) + } + + var highest flow.Header + err = operation.RetrieveHeader(blockID, &highest)(txn) + if err != nil { + return fmt.Errorf("cannot retrieve executed header: %w", err) + } + + if header.Height <= highest.Height { + return nil + } + err = operation.UpdateExecutedBlock(header.ID())(txn) + if err != nil { + return fmt.Errorf("cannot update highest executed block: %w", err) + } + + return nil + } +} + +// GetHighestExecutedBlock retrieves the height and ID of the latest block executed by this node. +// Returns storage.ErrNotFound if no latest executed block has been stored. +func GetHighestExecutedBlock(height *uint64, blockID *flow.Identifier) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + var highest flow.Header + err := operation.RetrieveExecutedBlock(blockID)(tx) + if err != nil { + return fmt.Errorf("could not lookup executed block %v: %w", blockID, err) + } + err = operation.RetrieveHeader(*blockID, &highest)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return fmt.Errorf("unexpected: latest executed block does not exist in storage: %s", err.Error()) + } + return fmt.Errorf("could not retrieve executed header %v: %w", blockID, err) + } + *height = highest.Height + return nil + } +} diff --git a/storage/pebble/procedure/executed_test.go b/storage/pebble/procedure/executed_test.go new file mode 100644 index 00000000000..ba776c17d97 --- /dev/null +++ b/storage/pebble/procedure/executed_test.go @@ -0,0 +1,91 @@ +package procedure + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertExecuted(t *testing.T) { + chain, _, _ := unittest.ChainFixture(6) + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + t.Run("setup and bootstrap", func(t *testing.T) { + for _, block := range chain { + require.NoError(t, db.Update(operation.InsertHeader(block.Header.ID(), block.Header))) + } + + root := chain[0].Header + require.NoError(t, + db.Update(operation.InsertExecutedBlock(root.ID())), + ) + + var height uint64 + var blockID flow.Identifier + require.NoError(t, + db.View(GetHighestExecutedBlock(&height, &blockID)), + ) + + require.Equal(t, root.ID(), blockID) + require.Equal(t, root.Height, height) + }) + + t.Run("insert and get", func(t *testing.T) { + header1 := chain[1].Header + require.NoError(t, + db.Update(UpdateHighestExecutedBlockIfHigher(header1)), + ) + + var height uint64 + var blockID flow.Identifier + require.NoError(t, + db.View(GetHighestExecutedBlock(&height, &blockID)), + ) + + require.Equal(t, header1.ID(), blockID) + require.Equal(t, header1.Height, height) + }) + + t.Run("insert more and get highest", func(t *testing.T) { + header2 := chain[2].Header + header3 := chain[3].Header + require.NoError(t, + db.Update(UpdateHighestExecutedBlockIfHigher(header2)), + ) + require.NoError(t, + db.Update(UpdateHighestExecutedBlockIfHigher(header3)), + ) + var height uint64 + var blockID flow.Identifier + require.NoError(t, + db.View(GetHighestExecutedBlock(&height, &blockID)), + ) + + require.Equal(t, header3.ID(), blockID) + require.Equal(t, header3.Height, height) + }) + + t.Run("insert lower height later and get highest", func(t *testing.T) { + header5 := chain[5].Header + header4 := chain[4].Header + require.NoError(t, + db.Update(UpdateHighestExecutedBlockIfHigher(header5)), + ) + require.NoError(t, + db.Update(UpdateHighestExecutedBlockIfHigher(header4)), + ) + var height uint64 + var blockID flow.Identifier + require.NoError(t, + db.View(GetHighestExecutedBlock(&height, &blockID)), + ) + + require.Equal(t, header5.ID(), blockID) + require.Equal(t, header5.Height, height) + }) + }) +} diff --git a/storage/pebble/procedure/index.go b/storage/pebble/procedure/index.go new file mode 100644 index 00000000000..a1a99127346 --- /dev/null +++ b/storage/pebble/procedure/index.go @@ -0,0 +1,65 @@ +package procedure + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage/badger/operation" +) + +func InsertIndex(blockID flow.Identifier, index *flow.Index) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + err := operation.IndexPayloadGuarantees(blockID, index.CollectionIDs)(tx) + if err != nil { + return fmt.Errorf("could not store guarantee index: %w", err) + } + err = operation.IndexPayloadSeals(blockID, index.SealIDs)(tx) + if err != nil { + return fmt.Errorf("could not store seal index: %w", err) + } + err = operation.IndexPayloadReceipts(blockID, index.ReceiptIDs)(tx) + if err != nil { + return fmt.Errorf("could not store receipts index: %w", err) + } + err = operation.IndexPayloadResults(blockID, index.ResultIDs)(tx) + if err != nil { + return fmt.Errorf("could not store results index: %w", err) + } + return nil + } +} + +func RetrieveIndex(blockID flow.Identifier, index *flow.Index) func(tx *badger.Txn) error { + return func(tx *badger.Txn) error { + var collIDs []flow.Identifier + err := operation.LookupPayloadGuarantees(blockID, &collIDs)(tx) + if err != nil { + return fmt.Errorf("could not retrieve guarantee index: %w", err) + } + var sealIDs []flow.Identifier + err = operation.LookupPayloadSeals(blockID, &sealIDs)(tx) + if err != nil { + return fmt.Errorf("could not retrieve seal index: %w", err) + } + var receiptIDs []flow.Identifier + err = operation.LookupPayloadReceipts(blockID, &receiptIDs)(tx) + if err != nil { + return fmt.Errorf("could not retrieve receipts index: %w", err) + } + var resultsIDs []flow.Identifier + err = operation.LookupPayloadResults(blockID, &resultsIDs)(tx) + if err != nil { + return fmt.Errorf("could not retrieve results index: %w", err) + } + + *index = flow.Index{ + CollectionIDs: collIDs, + SealIDs: sealIDs, + ReceiptIDs: receiptIDs, + ResultIDs: resultsIDs, + } + return nil + } +} diff --git a/storage/pebble/procedure/index_test.go b/storage/pebble/procedure/index_test.go new file mode 100644 index 00000000000..77a3c32bc9b --- /dev/null +++ b/storage/pebble/procedure/index_test.go @@ -0,0 +1,27 @@ +package procedure + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInsertRetrieveIndex(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + blockID := unittest.IdentifierFixture() + index := unittest.IndexFixture() + + err := db.Update(InsertIndex(blockID, index)) + require.NoError(t, err) + + var retrieved flow.Index + err = db.View(RetrieveIndex(blockID, &retrieved)) + require.NoError(t, err) + + require.Equal(t, index, &retrieved) + }) +} diff --git a/storage/pebble/qcs.go b/storage/pebble/qcs.go new file mode 100644 index 00000000000..856595184d4 --- /dev/null +++ b/storage/pebble/qcs.go @@ -0,0 +1,64 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// QuorumCertificates implements persistent storage for quorum certificates. +type QuorumCertificates struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.QuorumCertificate] +} + +var _ storage.QuorumCertificates = (*QuorumCertificates)(nil) + +// NewQuorumCertificates Creates QuorumCertificates instance which is a database of quorum certificates +// which supports storing, caching and retrieving by block ID. +func NewQuorumCertificates(collector module.CacheMetrics, db *badger.DB, cacheSize uint) *QuorumCertificates { + store := func(_ flow.Identifier, qc *flow.QuorumCertificate) func(*transaction.Tx) error { + return transaction.WithTx(operation.InsertQuorumCertificate(qc)) + } + + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.QuorumCertificate, error) { + return func(tx *badger.Txn) (*flow.QuorumCertificate, error) { + var qc flow.QuorumCertificate + err := operation.RetrieveQuorumCertificate(blockID, &qc)(tx) + return &qc, err + } + } + + return &QuorumCertificates{ + db: db, + cache: newCache[flow.Identifier, *flow.QuorumCertificate](collector, metrics.ResourceQC, + withLimit[flow.Identifier, *flow.QuorumCertificate](cacheSize), + withStore(store), + withRetrieve(retrieve)), + } +} + +func (q *QuorumCertificates) StoreTx(qc *flow.QuorumCertificate) func(*transaction.Tx) error { + return q.cache.PutTx(qc.BlockID, qc) +} + +func (q *QuorumCertificates) ByBlockID(blockID flow.Identifier) (*flow.QuorumCertificate, error) { + tx := q.db.NewTransaction(false) + defer tx.Discard() + return q.retrieveTx(blockID)(tx) +} + +func (q *QuorumCertificates) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.QuorumCertificate, error) { + return func(tx *badger.Txn) (*flow.QuorumCertificate, error) { + val, err := q.cache.Get(blockID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} diff --git a/storage/pebble/qcs_test.go b/storage/pebble/qcs_test.go new file mode 100644 index 00000000000..51cb0bc8a86 --- /dev/null +++ b/storage/pebble/qcs_test.go @@ -0,0 +1,70 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestQuorumCertificates_StoreTx tests storing and retrieving of QC. +func TestQuorumCertificates_StoreTx(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewQuorumCertificates(metrics, db, 10) + qc := unittest.QuorumCertificateFixture() + + err := operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(qc)) + require.NoError(t, err) + + actual, err := store.ByBlockID(qc.BlockID) + require.NoError(t, err) + + require.Equal(t, qc, actual) + }) +} + +// TestQuorumCertificates_StoreTx_OtherQC checks if storing other QC for same blockID results in +// expected storage error and already stored value is not overwritten. +func TestQuorumCertificates_StoreTx_OtherQC(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewQuorumCertificates(metrics, db, 10) + qc := unittest.QuorumCertificateFixture() + otherQC := unittest.QuorumCertificateFixture(func(otherQC *flow.QuorumCertificate) { + otherQC.View = qc.View + otherQC.BlockID = qc.BlockID + }) + + err := operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(qc)) + require.NoError(t, err) + + err = operation.RetryOnConflictTx(db, transaction.Update, store.StoreTx(otherQC)) + require.ErrorIs(t, err, storage.ErrAlreadyExists) + + actual, err := store.ByBlockID(otherQC.BlockID) + require.NoError(t, err) + + require.Equal(t, qc, actual) + }) +} + +// TestQuorumCertificates_ByBlockID that ByBlockID returns correct sentinel error if no QC for given block ID has been found +func TestQuorumCertificates_ByBlockID(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewQuorumCertificates(metrics, db, 10) + + actual, err := store.ByBlockID(unittest.IdentifierFixture()) + require.ErrorIs(t, err, storage.ErrNotFound) + require.Nil(t, actual) + }) +} diff --git a/storage/pebble/receipts.go b/storage/pebble/receipts.go new file mode 100644 index 00000000000..b92c3961048 --- /dev/null +++ b/storage/pebble/receipts.go @@ -0,0 +1,152 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// ExecutionReceipts implements storage for execution receipts. +type ExecutionReceipts struct { + db *badger.DB + results *ExecutionResults + cache *Cache[flow.Identifier, *flow.ExecutionReceipt] +} + +// NewExecutionReceipts Creates ExecutionReceipts instance which is a database of receipts which +// supports storing and indexing receipts by receipt ID and block ID. +func NewExecutionReceipts(collector module.CacheMetrics, db *badger.DB, results *ExecutionResults, cacheSize uint) *ExecutionReceipts { + store := func(receiptTD flow.Identifier, receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { + receiptID := receipt.ID() + + // assemble DB operations to store result (no execution) + storeResultOps := results.store(&receipt.ExecutionResult) + // assemble DB operations to index receipt (no execution) + storeReceiptOps := transaction.WithTx(operation.SkipDuplicates(operation.InsertExecutionReceiptMeta(receiptID, receipt.Meta()))) + // assemble DB operations to index receipt by the block it computes (no execution) + indexReceiptOps := transaction.WithTx(operation.SkipDuplicates( + operation.IndexExecutionReceipts(receipt.ExecutionResult.BlockID, receiptID), + )) + + return func(tx *transaction.Tx) error { + err := storeResultOps(tx) // execute operations to store results + if err != nil { + return fmt.Errorf("could not store result: %w", err) + } + err = storeReceiptOps(tx) // execute operations to store receipt-specific meta-data + if err != nil { + return fmt.Errorf("could not store receipt metadata: %w", err) + } + err = indexReceiptOps(tx) + if err != nil { + return fmt.Errorf("could not index receipt by the block it computes: %w", err) + } + return nil + } + } + + retrieve := func(receiptID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + var meta flow.ExecutionReceiptMeta + err := operation.RetrieveExecutionReceiptMeta(receiptID, &meta)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve receipt meta: %w", err) + } + result, err := results.byID(meta.ResultID)(tx) + if err != nil { + return nil, fmt.Errorf("could not retrieve result: %w", err) + } + return flow.ExecutionReceiptFromMeta(meta, *result), nil + } + } + + return &ExecutionReceipts{ + db: db, + results: results, + cache: newCache[flow.Identifier, *flow.ExecutionReceipt](collector, metrics.ResourceReceipt, + withLimit[flow.Identifier, *flow.ExecutionReceipt](cacheSize), + withStore(store), + withRetrieve(retrieve)), + } +} + +// storeMyReceipt assembles the operations to store an arbitrary receipt. +func (r *ExecutionReceipts) storeTx(receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { + return r.cache.PutTx(receipt.ID(), receipt) +} + +func (r *ExecutionReceipts) byID(receiptID flow.Identifier) func(*badger.Txn) (*flow.ExecutionReceipt, error) { + retrievalOps := r.cache.Get(receiptID) // assemble DB operations to retrieve receipt (no execution) + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + val, err := retrievalOps(tx) // execute operations to retrieve receipt + if err != nil { + return nil, err + } + return val, nil + } +} + +func (r *ExecutionReceipts) byBlockID(blockID flow.Identifier) func(*badger.Txn) ([]*flow.ExecutionReceipt, error) { + return func(tx *badger.Txn) ([]*flow.ExecutionReceipt, error) { + var receiptIDs []flow.Identifier + err := operation.LookupExecutionReceipts(blockID, &receiptIDs)(tx) + if err != nil && !errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("could not find receipt index for block: %w", err) + } + + var receipts []*flow.ExecutionReceipt + for _, id := range receiptIDs { + receipt, err := r.byID(id)(tx) + if err != nil { + return nil, fmt.Errorf("could not find receipt with id %v: %w", id, err) + } + receipts = append(receipts, receipt) + } + return receipts, nil + } +} + +func (r *ExecutionReceipts) Store(receipt *flow.ExecutionReceipt) error { + return operation.RetryOnConflictTx(r.db, transaction.Update, r.storeTx(receipt)) +} + +func (r *ExecutionReceipts) BatchStore(receipt *flow.ExecutionReceipt, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + + err := r.results.BatchStore(&receipt.ExecutionResult, batch) + if err != nil { + return fmt.Errorf("cannot batch store execution result inside execution receipt batch store: %w", err) + } + + err = operation.BatchInsertExecutionReceiptMeta(receipt.ID(), receipt.Meta())(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch store execution meta inside execution receipt batch store: %w", err) + } + + err = operation.BatchIndexExecutionReceipts(receipt.ExecutionResult.BlockID, receipt.ID())(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch index execution receipt inside execution receipt batch store: %w", err) + } + + return nil +} + +func (r *ExecutionReceipts) ByID(receiptID flow.Identifier) (*flow.ExecutionReceipt, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byID(receiptID)(tx) +} + +func (r *ExecutionReceipts) ByBlockID(blockID flow.Identifier) (flow.ExecutionReceiptList, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byBlockID(blockID)(tx) +} diff --git a/storage/pebble/receipts_test.go b/storage/pebble/receipts_test.go new file mode 100644 index 00000000000..03b8420258e --- /dev/null +++ b/storage/pebble/receipts_test.go @@ -0,0 +1,146 @@ +package badger_test + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestExecutionReceiptsStorage(t *testing.T) { + withStore := func(t *testing.T, f func(store *bstorage.ExecutionReceipts)) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + results := bstorage.NewExecutionResults(metrics, db) + store := bstorage.NewExecutionReceipts(metrics, db, results, bstorage.DefaultCacheSize) + f(store) + }) + } + + t.Run("get empty", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block := unittest.BlockFixture() + receipts, err := store.ByBlockID(block.ID()) + require.NoError(t, err) + require.Equal(t, 0, len(receipts)) + }) + }) + + t.Run("store one get one", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block := unittest.BlockFixture() + receipt1 := unittest.ReceiptForBlockFixture(&block) + + err := store.Store(receipt1) + require.NoError(t, err) + + actual, err := store.ByID(receipt1.ID()) + require.NoError(t, err) + + require.Equal(t, receipt1, actual) + + receipts, err := store.ByBlockID(block.ID()) + require.NoError(t, err) + + require.Equal(t, flow.ExecutionReceiptList{receipt1}, receipts) + }) + }) + + t.Run("store two for the same block", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block := unittest.BlockFixture() + + executor1 := unittest.IdentifierFixture() + executor2 := unittest.IdentifierFixture() + + receipt1 := unittest.ReceiptForBlockExecutorFixture(&block, executor1) + receipt2 := unittest.ReceiptForBlockExecutorFixture(&block, executor2) + + err := store.Store(receipt1) + require.NoError(t, err) + + err = store.Store(receipt2) + require.NoError(t, err) + + receipts, err := store.ByBlockID(block.ID()) + require.NoError(t, err) + + require.ElementsMatch(t, []*flow.ExecutionReceipt{receipt1, receipt2}, receipts) + }) + }) + + t.Run("store two for different blocks", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block1 := unittest.BlockFixture() + block2 := unittest.BlockFixture() + + executor1 := unittest.IdentifierFixture() + executor2 := unittest.IdentifierFixture() + + receipt1 := unittest.ReceiptForBlockExecutorFixture(&block1, executor1) + receipt2 := unittest.ReceiptForBlockExecutorFixture(&block2, executor2) + + err := store.Store(receipt1) + require.NoError(t, err) + + err = store.Store(receipt2) + require.NoError(t, err) + + receipts1, err := store.ByBlockID(block1.ID()) + require.NoError(t, err) + + receipts2, err := store.ByBlockID(block2.ID()) + require.NoError(t, err) + + require.ElementsMatch(t, []*flow.ExecutionReceipt{receipt1}, receipts1) + require.ElementsMatch(t, []*flow.ExecutionReceipt{receipt2}, receipts2) + }) + }) + + t.Run("indexing duplicated receipts should be ok", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block1 := unittest.BlockFixture() + + executor1 := unittest.IdentifierFixture() + receipt1 := unittest.ReceiptForBlockExecutorFixture(&block1, executor1) + + err := store.Store(receipt1) + require.NoError(t, err) + + err = store.Store(receipt1) + require.NoError(t, err) + + receipts, err := store.ByBlockID(block1.ID()) + require.NoError(t, err) + + require.ElementsMatch(t, []*flow.ExecutionReceipt{receipt1}, receipts) + }) + }) + + t.Run("indexing receipt from the same executor for same block should succeed", func(t *testing.T) { + withStore(t, func(store *bstorage.ExecutionReceipts) { + block1 := unittest.BlockFixture() + + executor1 := unittest.IdentifierFixture() + + receipt1 := unittest.ReceiptForBlockExecutorFixture(&block1, executor1) + receipt2 := unittest.ReceiptForBlockExecutorFixture(&block1, executor1) + + err := store.Store(receipt1) + require.NoError(t, err) + + err = store.Store(receipt2) + require.NoError(t, err) + + receipts, err := store.ByBlockID(block1.ID()) + require.NoError(t, err) + + require.ElementsMatch(t, []*flow.ExecutionReceipt{receipt1, receipt2}, receipts) + }) + }) +} diff --git a/storage/pebble/results.go b/storage/pebble/results.go new file mode 100644 index 00000000000..d4d1a4525b0 --- /dev/null +++ b/storage/pebble/results.go @@ -0,0 +1,166 @@ +package badger + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// ExecutionResults implements persistent storage for execution results. +type ExecutionResults struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.ExecutionResult] +} + +var _ storage.ExecutionResults = (*ExecutionResults)(nil) + +func NewExecutionResults(collector module.CacheMetrics, db *badger.DB) *ExecutionResults { + + store := func(_ flow.Identifier, result *flow.ExecutionResult) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertExecutionResult(result))) + } + + retrieve := func(resultID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionResult, error) { + return func(tx *badger.Txn) (*flow.ExecutionResult, error) { + var result flow.ExecutionResult + err := operation.RetrieveExecutionResult(resultID, &result)(tx) + return &result, err + } + } + + res := &ExecutionResults{ + db: db, + cache: newCache[flow.Identifier, *flow.ExecutionResult](collector, metrics.ResourceResult, + withLimit[flow.Identifier, *flow.ExecutionResult](flow.DefaultTransactionExpiry+100), + withStore(store), + withRetrieve(retrieve)), + } + + return res +} + +func (r *ExecutionResults) store(result *flow.ExecutionResult) func(*transaction.Tx) error { + return r.cache.PutTx(result.ID(), result) +} + +func (r *ExecutionResults) byID(resultID flow.Identifier) func(*badger.Txn) (*flow.ExecutionResult, error) { + return func(tx *badger.Txn) (*flow.ExecutionResult, error) { + val, err := r.cache.Get(resultID)(tx) + if err != nil { + return nil, err + } + return val, nil + } +} + +func (r *ExecutionResults) byBlockID(blockID flow.Identifier) func(*badger.Txn) (*flow.ExecutionResult, error) { + return func(tx *badger.Txn) (*flow.ExecutionResult, error) { + var resultID flow.Identifier + err := operation.LookupExecutionResult(blockID, &resultID)(tx) + if err != nil { + return nil, fmt.Errorf("could not lookup execution result ID: %w", err) + } + return r.byID(resultID)(tx) + } +} + +func (r *ExecutionResults) index(blockID, resultID flow.Identifier, force bool) func(*transaction.Tx) error { + return func(tx *transaction.Tx) error { + err := transaction.WithTx(operation.IndexExecutionResult(blockID, resultID))(tx) + if err == nil { + return nil + } + + if !errors.Is(err, storage.ErrAlreadyExists) { + return err + } + + if force { + return transaction.WithTx(operation.ReindexExecutionResult(blockID, resultID))(tx) + } + + // when trying to index a result for a block, and there is already a result indexed for this block, + // double check if the indexed result is the same + var storedResultID flow.Identifier + err = transaction.WithTx(operation.LookupExecutionResult(blockID, &storedResultID))(tx) + if err != nil { + return fmt.Errorf("there is a result stored already, but cannot retrieve it: %w", err) + } + + if storedResultID != resultID { + return fmt.Errorf("storing result that is different from the already stored one for block: %v, storing result: %v, stored result: %v. %w", + blockID, resultID, storedResultID, storage.ErrDataMismatch) + } + + return nil + } +} + +func (r *ExecutionResults) Store(result *flow.ExecutionResult) error { + return operation.RetryOnConflictTx(r.db, transaction.Update, r.store(result)) +} + +func (r *ExecutionResults) BatchStore(result *flow.ExecutionResult, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return operation.BatchInsertExecutionResult(result)(writeBatch) +} + +func (r *ExecutionResults) BatchIndex(blockID flow.Identifier, resultID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return operation.BatchIndexExecutionResult(blockID, resultID)(writeBatch) +} + +func (r *ExecutionResults) ByID(resultID flow.Identifier) (*flow.ExecutionResult, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byID(resultID)(tx) +} + +func (r *ExecutionResults) ByIDTx(resultID flow.Identifier) func(*transaction.Tx) (*flow.ExecutionResult, error) { + return func(tx *transaction.Tx) (*flow.ExecutionResult, error) { + result, err := r.byID(resultID)(tx.DBTxn) + return result, err + } +} + +func (r *ExecutionResults) Index(blockID flow.Identifier, resultID flow.Identifier) error { + err := operation.RetryOnConflictTx(r.db, transaction.Update, r.index(blockID, resultID, false)) + if err != nil { + return fmt.Errorf("could not index execution result: %w", err) + } + return nil +} + +func (r *ExecutionResults) ForceIndex(blockID flow.Identifier, resultID flow.Identifier) error { + err := operation.RetryOnConflictTx(r.db, transaction.Update, r.index(blockID, resultID, true)) + if err != nil { + return fmt.Errorf("could not index execution result: %w", err) + } + return nil +} + +func (r *ExecutionResults) ByBlockID(blockID flow.Identifier) (*flow.ExecutionResult, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + return r.byBlockID(blockID)(tx) +} + +func (r *ExecutionResults) RemoveIndexByBlockID(blockID flow.Identifier) error { + return r.db.Update(operation.SkipNonExist(operation.RemoveExecutionResultIndex(blockID))) +} + +// BatchRemoveIndexByBlockID removes blockID-to-executionResultID index entries keyed by blockID in a provided batch. +// No errors are expected during normal operation, even if no entries are matched. +// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. +func (r *ExecutionResults) BatchRemoveIndexByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return operation.BatchRemoveExecutionResultIndex(blockID)(writeBatch) +} diff --git a/storage/pebble/results_test.go b/storage/pebble/results_test.go new file mode 100644 index 00000000000..a23c8bf7232 --- /dev/null +++ b/storage/pebble/results_test.go @@ -0,0 +1,137 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestResultStoreAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewExecutionResults(metrics, db) + + result := unittest.ExecutionResultFixture() + blockID := unittest.IdentifierFixture() + err := store.Store(result) + require.NoError(t, err) + + err = store.Index(blockID, result.ID()) + require.NoError(t, err) + + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + + require.Equal(t, result, actual) + }) +} + +func TestResultStoreTwice(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewExecutionResults(metrics, db) + + result := unittest.ExecutionResultFixture() + blockID := unittest.IdentifierFixture() + err := store.Store(result) + require.NoError(t, err) + + err = store.Index(blockID, result.ID()) + require.NoError(t, err) + + err = store.Store(result) + require.NoError(t, err) + + err = store.Index(blockID, result.ID()) + require.NoError(t, err) + }) +} + +func TestResultBatchStoreTwice(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewExecutionResults(metrics, db) + + result := unittest.ExecutionResultFixture() + blockID := unittest.IdentifierFixture() + + batch := bstorage.NewBatch(db) + err := store.BatchStore(result, batch) + require.NoError(t, err) + + err = store.BatchIndex(blockID, result.ID(), batch) + require.NoError(t, err) + + require.NoError(t, batch.Flush()) + + batch = bstorage.NewBatch(db) + err = store.BatchStore(result, batch) + require.NoError(t, err) + + err = store.BatchIndex(blockID, result.ID(), batch) + require.NoError(t, err) + + require.NoError(t, batch.Flush()) + }) +} + +func TestResultStoreTwoDifferentResultsShouldFail(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewExecutionResults(metrics, db) + + result1 := unittest.ExecutionResultFixture() + result2 := unittest.ExecutionResultFixture() + blockID := unittest.IdentifierFixture() + err := store.Store(result1) + require.NoError(t, err) + + err = store.Index(blockID, result1.ID()) + require.NoError(t, err) + + // we can store a different result, but we can't index + // a different result for that block, because it will mean + // one block has two different results. + err = store.Store(result2) + require.NoError(t, err) + + err = store.Index(blockID, result2.ID()) + require.Error(t, err) + require.True(t, errors.Is(err, storage.ErrDataMismatch)) + }) +} + +func TestResultStoreForceIndexOverridesMapping(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewExecutionResults(metrics, db) + + result1 := unittest.ExecutionResultFixture() + result2 := unittest.ExecutionResultFixture() + blockID := unittest.IdentifierFixture() + err := store.Store(result1) + require.NoError(t, err) + err = store.Index(blockID, result1.ID()) + require.NoError(t, err) + + err = store.Store(result2) + require.NoError(t, err) + + // force index + err = store.ForceIndex(blockID, result2.ID()) + require.NoError(t, err) + + // retrieve index to make sure it points to second ER now + byBlockID, err := store.ByBlockID(blockID) + + require.Equal(t, result2, byBlockID) + require.NoError(t, err) + }) +} diff --git a/storage/pebble/seals.go b/storage/pebble/seals.go new file mode 100644 index 00000000000..5ae5cbe71af --- /dev/null +++ b/storage/pebble/seals.go @@ -0,0 +1,94 @@ +// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED + +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +type Seals struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.Seal] +} + +func NewSeals(collector module.CacheMetrics, db *badger.DB) *Seals { + + store := func(sealID flow.Identifier, seal *flow.Seal) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertSeal(sealID, seal))) + } + + retrieve := func(sealID flow.Identifier) func(*badger.Txn) (*flow.Seal, error) { + return func(tx *badger.Txn) (*flow.Seal, error) { + var seal flow.Seal + err := operation.RetrieveSeal(sealID, &seal)(tx) + return &seal, err + } + } + + s := &Seals{ + db: db, + cache: newCache[flow.Identifier, *flow.Seal](collector, metrics.ResourceSeal, + withLimit[flow.Identifier, *flow.Seal](flow.DefaultTransactionExpiry+100), + withStore(store), + withRetrieve(retrieve)), + } + + return s +} + +func (s *Seals) storeTx(seal *flow.Seal) func(*transaction.Tx) error { + return s.cache.PutTx(seal.ID(), seal) +} + +func (s *Seals) retrieveTx(sealID flow.Identifier) func(*badger.Txn) (*flow.Seal, error) { + return func(tx *badger.Txn) (*flow.Seal, error) { + val, err := s.cache.Get(sealID)(tx) + if err != nil { + return nil, err + } + return val, err + } +} + +func (s *Seals) Store(seal *flow.Seal) error { + return operation.RetryOnConflictTx(s.db, transaction.Update, s.storeTx(seal)) +} + +func (s *Seals) ByID(sealID flow.Identifier) (*flow.Seal, error) { + tx := s.db.NewTransaction(false) + defer tx.Discard() + return s.retrieveTx(sealID)(tx) +} + +// HighestInFork retrieves the highest seal that was included in the +// fork up to (and including) blockID. This method should return a seal +// for any block known to the node. Returns storage.ErrNotFound if +// blockID is unknown. +func (s *Seals) HighestInFork(blockID flow.Identifier) (*flow.Seal, error) { + var sealID flow.Identifier + err := s.db.View(operation.LookupLatestSealAtBlock(blockID, &sealID)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve seal for fork with head %x: %w", blockID, err) + } + return s.ByID(sealID) +} + +// FinalizedSealForBlock returns the seal for the given block, only if that seal +// has been included in a finalized block. +// Returns storage.ErrNotFound if the block is unknown or unsealed. +func (s *Seals) FinalizedSealForBlock(blockID flow.Identifier) (*flow.Seal, error) { + var sealID flow.Identifier + err := s.db.View(operation.LookupBySealedBlockID(blockID, &sealID)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve seal for block %x: %w", blockID, err) + } + return s.ByID(sealID) +} diff --git a/storage/pebble/seals_test.go b/storage/pebble/seals_test.go new file mode 100644 index 00000000000..5e700941c0b --- /dev/null +++ b/storage/pebble/seals_test.go @@ -0,0 +1,101 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestRetrieveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewSeals(metrics, db) + + _, err := store.ByID(unittest.IdentifierFixture()) + require.True(t, errors.Is(err, storage.ErrNotFound)) + + _, err = store.HighestInFork(unittest.IdentifierFixture()) + require.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} + +// TestSealStoreRetrieve verifies that a seal can be stored and retrieved by its ID +func TestSealStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewSeals(metrics, db) + + expected := unittest.Seal.Fixture() + // store seal + err := store.Store(expected) + require.NoError(t, err) + + // retrieve seal + seal, err := store.ByID(expected.ID()) + require.NoError(t, err) + require.Equal(t, expected, seal) + }) +} + +// TestSealIndexAndRetrieve verifies that: +// - for a block, we can store (aka index) the latest sealed block along this fork. +// +// Note: indexing the seal for a block is currently implemented only through a direct +// Badger operation. The Seals mempool only supports retrieving the latest sealed block. +func TestSealIndexAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewSeals(metrics, db) + + expectedSeal := unittest.Seal.Fixture() + blockID := unittest.IdentifierFixture() + + // store the seal first + err := store.Store(expectedSeal) + require.NoError(t, err) + + // index the seal ID for the heighest sealed block in this fork + err = operation.RetryOnConflict(db.Update, operation.IndexLatestSealAtBlock(blockID, expectedSeal.ID())) + require.NoError(t, err) + + // retrieve latest seal + seal, err := store.HighestInFork(blockID) + require.NoError(t, err) + require.Equal(t, expectedSeal, seal) + }) +} + +// TestSealedBlockIndexAndRetrieve checks after indexing a seal by a sealed block ID, it can be +// retrieved by the sealed block ID +func TestSealedBlockIndexAndRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewSeals(metrics, db) + + expectedSeal := unittest.Seal.Fixture() + blockID := unittest.IdentifierFixture() + expectedSeal.BlockID = blockID + + // store the seal first + err := store.Store(expectedSeal) + require.NoError(t, err) + + // index the seal ID for the highest sealed block in this fork + err = operation.RetryOnConflict(db.Update, operation.IndexFinalizedSealByBlockID(expectedSeal.BlockID, expectedSeal.ID())) + require.NoError(t, err) + + // retrieve latest seal + seal, err := store.FinalizedSealForBlock(blockID) + require.NoError(t, err) + require.Equal(t, expectedSeal, seal) + }) +} diff --git a/storage/pebble/transaction_results.go b/storage/pebble/transaction_results.go new file mode 100644 index 00000000000..1aca9e63b11 --- /dev/null +++ b/storage/pebble/transaction_results.go @@ -0,0 +1,235 @@ +package badger + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +var _ storage.TransactionResults = (*TransactionResults)(nil) + +type TransactionResults struct { + db *badger.DB + cache *Cache[string, flow.TransactionResult] + indexCache *Cache[string, flow.TransactionResult] + blockCache *Cache[string, []flow.TransactionResult] +} + +func KeyFromBlockIDTransactionID(blockID flow.Identifier, txID flow.Identifier) string { + return fmt.Sprintf("%x%x", blockID, txID) +} + +func KeyFromBlockIDIndex(blockID flow.Identifier, txIndex uint32) string { + idData := make([]byte, 4) //uint32 fits into 4 bytes + binary.BigEndian.PutUint32(idData, txIndex) + return fmt.Sprintf("%x%x", blockID, idData) +} + +func KeyFromBlockID(blockID flow.Identifier) string { + return blockID.String() +} + +func KeyToBlockIDTransactionID(key string) (flow.Identifier, flow.Identifier, error) { + blockIDStr := key[:64] + txIDStr := key[64:] + blockID, err := flow.HexStringToIdentifier(blockIDStr) + if err != nil { + return flow.ZeroID, flow.ZeroID, fmt.Errorf("could not get block ID: %w", err) + } + + txID, err := flow.HexStringToIdentifier(txIDStr) + if err != nil { + return flow.ZeroID, flow.ZeroID, fmt.Errorf("could not get transaction id: %w", err) + } + + return blockID, txID, nil +} + +func KeyToBlockIDIndex(key string) (flow.Identifier, uint32, error) { + blockIDStr := key[:64] + indexStr := key[64:] + blockID, err := flow.HexStringToIdentifier(blockIDStr) + if err != nil { + return flow.ZeroID, 0, fmt.Errorf("could not get block ID: %w", err) + } + + txIndexBytes, err := hex.DecodeString(indexStr) + if err != nil { + return flow.ZeroID, 0, fmt.Errorf("could not get transaction index: %w", err) + } + if len(txIndexBytes) != 4 { + return flow.ZeroID, 0, fmt.Errorf("could not get transaction index - invalid length: %d", len(txIndexBytes)) + } + + txIndex := binary.BigEndian.Uint32(txIndexBytes) + + return blockID, txIndex, nil +} + +func KeyToBlockID(key string) (flow.Identifier, error) { + + blockID, err := flow.HexStringToIdentifier(key) + if err != nil { + return flow.ZeroID, fmt.Errorf("could not get block ID: %w", err) + } + + return blockID, err +} + +func NewTransactionResults(collector module.CacheMetrics, db *badger.DB, transactionResultsCacheSize uint) *TransactionResults { + retrieve := func(key string) func(tx *badger.Txn) (flow.TransactionResult, error) { + var txResult flow.TransactionResult + return func(tx *badger.Txn) (flow.TransactionResult, error) { + + blockID, txID, err := KeyToBlockIDTransactionID(key) + if err != nil { + return flow.TransactionResult{}, fmt.Errorf("could not convert key: %w", err) + } + + err = operation.RetrieveTransactionResult(blockID, txID, &txResult)(tx) + if err != nil { + return flow.TransactionResult{}, handleError(err, flow.TransactionResult{}) + } + return txResult, nil + } + } + retrieveIndex := func(key string) func(tx *badger.Txn) (flow.TransactionResult, error) { + var txResult flow.TransactionResult + return func(tx *badger.Txn) (flow.TransactionResult, error) { + + blockID, txIndex, err := KeyToBlockIDIndex(key) + if err != nil { + return flow.TransactionResult{}, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.RetrieveTransactionResultByIndex(blockID, txIndex, &txResult)(tx) + if err != nil { + return flow.TransactionResult{}, handleError(err, flow.TransactionResult{}) + } + return txResult, nil + } + } + retrieveForBlock := func(key string) func(tx *badger.Txn) ([]flow.TransactionResult, error) { + var txResults []flow.TransactionResult + return func(tx *badger.Txn) ([]flow.TransactionResult, error) { + + blockID, err := KeyToBlockID(key) + if err != nil { + return nil, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.LookupTransactionResultsByBlockIDUsingIndex(blockID, &txResults)(tx) + if err != nil { + return nil, handleError(err, flow.TransactionResult{}) + } + return txResults, nil + } + } + return &TransactionResults{ + db: db, + cache: newCache[string, flow.TransactionResult](collector, metrics.ResourceTransactionResults, + withLimit[string, flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResult]), + withRetrieve(retrieve), + ), + indexCache: newCache[string, flow.TransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResult]), + withRetrieve(retrieveIndex), + ), + blockCache: newCache[string, []flow.TransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, []flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, []flow.TransactionResult]), + withRetrieve(retrieveForBlock), + ), + } +} + +// BatchStore will store the transaction results for the given block ID in a batch +func (tr *TransactionResults) BatchStore(blockID flow.Identifier, transactionResults []flow.TransactionResult, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + + for i, result := range transactionResults { + err := operation.BatchInsertTransactionResult(blockID, &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch insert tx result: %w", err) + } + + err = operation.BatchIndexTransactionResult(blockID, uint32(i), &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch index tx result: %w", err) + } + } + + batch.OnSucceed(func() { + for i, result := range transactionResults { + key := KeyFromBlockIDTransactionID(blockID, result.TransactionID) + // cache for each transaction, so that it's faster to retrieve + tr.cache.Insert(key, result) + + index := uint32(i) + + keyIndex := KeyFromBlockIDIndex(blockID, index) + tr.indexCache.Insert(keyIndex, result) + } + + key := KeyFromBlockID(blockID) + tr.blockCache.Insert(key, transactionResults) + }) + return nil +} + +// ByBlockIDTransactionID returns the runtime transaction result for the given block ID and transaction ID +func (tr *TransactionResults) ByBlockIDTransactionID(blockID flow.Identifier, txID flow.Identifier) (*flow.TransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDTransactionID(blockID, txID) + transactionResult, err := tr.cache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResult, nil +} + +// ByBlockIDTransactionIndex returns the runtime transaction result for the given block ID and transaction index +func (tr *TransactionResults) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.TransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDIndex(blockID, txIndex) + transactionResult, err := tr.indexCache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResult, nil +} + +// ByBlockID gets all transaction results for a block, ordered by transaction index +func (tr *TransactionResults) ByBlockID(blockID flow.Identifier) ([]flow.TransactionResult, error) { + tx := tr.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockID(blockID) + transactionResults, err := tr.blockCache.Get(key)(tx) + if err != nil { + return nil, err + } + return transactionResults, nil +} + +// RemoveByBlockID removes transaction results by block ID +func (tr *TransactionResults) RemoveByBlockID(blockID flow.Identifier) error { + return tr.db.Update(operation.RemoveTransactionResultsByBlockID(blockID)) +} + +// BatchRemoveByBlockID batch removes transaction results by block ID +func (tr *TransactionResults) BatchRemoveByBlockID(blockID flow.Identifier, batch storage.BatchStorage) error { + writeBatch := batch.GetWriter() + return tr.db.View(operation.BatchRemoveTransactionResultsByBlockID(blockID, writeBatch)) +} diff --git a/storage/pebble/transaction_results_test.go b/storage/pebble/transaction_results_test.go new file mode 100644 index 00000000000..5ba30d74414 --- /dev/null +++ b/storage/pebble/transaction_results_test.go @@ -0,0 +1,105 @@ +package badger_test + +import ( + "fmt" + mathRand "math/rand" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + bstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestBatchStoringTransactionResults(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewTransactionResults(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + txResults := make([]flow.TransactionResult, 0) + for i := 0; i < 10; i++ { + txID := unittest.IdentifierFixture() + expected := flow.TransactionResult{ + TransactionID: txID, + ErrorMessage: fmt.Sprintf("a runtime error %d", i), + } + txResults = append(txResults, expected) + } + writeBatch := bstorage.NewBatch(db) + err := store.BatchStore(blockID, txResults, writeBatch) + require.NoError(t, err) + + err = writeBatch.Flush() + require.NoError(t, err) + + for _, txResult := range txResults { + actual, err := store.ByBlockIDTransactionID(blockID, txResult.TransactionID) + require.NoError(t, err) + assert.Equal(t, txResult, *actual) + } + + // test loading from database + newStore := bstorage.NewTransactionResults(metrics, db, 1000) + for _, txResult := range txResults { + actual, err := newStore.ByBlockIDTransactionID(blockID, txResult.TransactionID) + require.NoError(t, err) + assert.Equal(t, txResult, *actual) + } + + // check retrieving by index from both cache and db + for i := len(txResults) - 1; i >= 0; i-- { + actual, err := store.ByBlockIDTransactionIndex(blockID, uint32(i)) + require.NoError(t, err) + assert.Equal(t, txResults[i], *actual) + + actual, err = newStore.ByBlockIDTransactionIndex(blockID, uint32(i)) + require.NoError(t, err) + assert.Equal(t, txResults[i], *actual) + } + }) +} + +func TestReadingNotStoreTransaction(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewTransactionResults(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + txID := unittest.IdentifierFixture() + txIndex := rand.Uint32() + + _, err := store.ByBlockIDTransactionID(blockID, txID) + assert.ErrorIs(t, err, storage.ErrNotFound) + + _, err = store.ByBlockIDTransactionIndex(blockID, txIndex) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) +} + +func TestKeyConversion(t *testing.T) { + blockID := unittest.IdentifierFixture() + txID := unittest.IdentifierFixture() + key := bstorage.KeyFromBlockIDTransactionID(blockID, txID) + bID, tID, err := bstorage.KeyToBlockIDTransactionID(key) + require.NoError(t, err) + require.Equal(t, blockID, bID) + require.Equal(t, txID, tID) +} + +func TestIndexKeyConversion(t *testing.T) { + blockID := unittest.IdentifierFixture() + txIndex := mathRand.Uint32() + key := bstorage.KeyFromBlockIDIndex(blockID, txIndex) + bID, tID, err := bstorage.KeyToBlockIDIndex(key) + require.NoError(t, err) + require.Equal(t, blockID, bID) + require.Equal(t, txIndex, tID) +} diff --git a/storage/pebble/transactions.go b/storage/pebble/transactions.go new file mode 100644 index 00000000000..eeca9c9477e --- /dev/null +++ b/storage/pebble/transactions.go @@ -0,0 +1,68 @@ +package badger + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" +) + +// Transactions ... +type Transactions struct { + db *badger.DB + cache *Cache[flow.Identifier, *flow.TransactionBody] +} + +// NewTransactions ... +func NewTransactions(cacheMetrics module.CacheMetrics, db *badger.DB) *Transactions { + store := func(txID flow.Identifier, flowTX *flow.TransactionBody) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertTransaction(txID, flowTX))) + } + + retrieve := func(txID flow.Identifier) func(tx *badger.Txn) (*flow.TransactionBody, error) { + return func(tx *badger.Txn) (*flow.TransactionBody, error) { + var flowTx flow.TransactionBody + err := operation.RetrieveTransaction(txID, &flowTx)(tx) + return &flowTx, err + } + } + + t := &Transactions{ + db: db, + cache: newCache[flow.Identifier, *flow.TransactionBody](cacheMetrics, metrics.ResourceTransaction, + withLimit[flow.Identifier, *flow.TransactionBody](flow.DefaultTransactionExpiry+100), + withStore(store), + withRetrieve(retrieve)), + } + + return t +} + +// Store ... +func (t *Transactions) Store(flowTx *flow.TransactionBody) error { + return operation.RetryOnConflictTx(t.db, transaction.Update, t.storeTx(flowTx)) +} + +// ByID ... +func (t *Transactions) ByID(txID flow.Identifier) (*flow.TransactionBody, error) { + tx := t.db.NewTransaction(false) + defer tx.Discard() + return t.retrieveTx(txID)(tx) +} + +func (t *Transactions) storeTx(flowTx *flow.TransactionBody) func(*transaction.Tx) error { + return t.cache.PutTx(flowTx.ID(), flowTx) +} + +func (t *Transactions) retrieveTx(txID flow.Identifier) func(*badger.Txn) (*flow.TransactionBody, error) { + return func(tx *badger.Txn) (*flow.TransactionBody, error) { + val, err := t.cache.Get(txID)(tx) + if err != nil { + return nil, err + } + return val, err + } +} diff --git a/storage/pebble/transactions_test.go b/storage/pebble/transactions_test.go new file mode 100644 index 00000000000..3b10a10dc5b --- /dev/null +++ b/storage/pebble/transactions_test.go @@ -0,0 +1,48 @@ +package badger_test + +import ( + "errors" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + badgerstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestTransactionStoreRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewTransactions(metrics, db) + + // store a transaction in db + expected := unittest.TransactionFixture() + err := store.Store(&expected.TransactionBody) + require.NoError(t, err) + + // retrieve the transaction by ID + actual, err := store.ByID(expected.ID()) + require.NoError(t, err) + assert.Equal(t, &expected.TransactionBody, actual) + + // re-insert the transaction - should be idempotent + err = store.Store(&expected.TransactionBody) + require.NoError(t, err) + }) +} + +func TestTransactionRetrieveWithoutStore(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := badgerstorage.NewTransactions(metrics, db) + + // attempt to get a invalid transaction + _, err := store.ByID(unittest.IdentifierFixture()) + assert.True(t, errors.Is(err, storage.ErrNotFound)) + }) +} diff --git a/storage/pebble/version_beacon.go b/storage/pebble/version_beacon.go new file mode 100644 index 00000000000..7300c2fc568 --- /dev/null +++ b/storage/pebble/version_beacon.go @@ -0,0 +1,43 @@ +package badger + +import ( + "errors" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +type VersionBeacons struct { + db *badger.DB +} + +var _ storage.VersionBeacons = (*VersionBeacons)(nil) + +func NewVersionBeacons(db *badger.DB) *VersionBeacons { + res := &VersionBeacons{ + db: db, + } + + return res +} + +func (r *VersionBeacons) Highest( + belowOrEqualTo uint64, +) (*flow.SealedVersionBeacon, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + + var beacon flow.SealedVersionBeacon + + err := operation.LookupLastVersionBeaconByHeight(belowOrEqualTo, &beacon)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, nil + } + return nil, err + } + return &beacon, nil +}