Added recovery mechanism for missing block result

packages/persistance/src/services/prisma/PrismaBlockStorage.ts

@@ -8,6 +8,7 @@ import {
   BlockStorage,
   BlockWithResult,
   BlockWithPreviousResult,
+  BlockWithMaybeResult,
 } from "@proto-kit/sequencer";
 import { filterNonNull, log } from "@proto-kit/common";
 import {
@@ -39,7 +40,7 @@ export class PrismaBlockStorage
 
   private async getBlockByQuery(
     where: { height: number } | { hash: string }
-  ): Promise<BlockWithResult | undefined> {
+  ): Promise<BlockWithMaybeResult | undefined> {
     const dbResult = await this.connection.prismaClient.block.findFirst({
       where,
       include: {
@@ -57,18 +58,15 @@ export class PrismaBlockStorage
     const transactions = dbResult.transactions.map<TransactionExecutionResult>(
       (txresult) => this.transactionResultMapper.mapIn([txresult, txresult.tx])
     );
-    if (dbResult.result === undefined || dbResult.result === null) {
-      throw new Error(
-        `No Metadata has been set for block ${JSON.stringify(where)} yet`
-      );
-    }
 
     return {
       block: {
         ...this.blockMapper.mapIn(dbResult),
         transactions,
       },
-      result: this.blockResultMapper.mapIn(dbResult.result),
+      result: dbResult.result
+        ? this.blockResultMapper.mapIn(dbResult.result)
+        : undefined,
     };
   }
 
@@ -169,7 +167,9 @@ export class PrismaBlockStorage
     return (result?._max.height ?? -1) + 1;
   }
 
-  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+  public async getLatestBlockAndResult(): Promise<
+    BlockWithMaybeResult | undefined
+  > {
     const latestBlock = await this.connection.prismaClient.$queryRaw<
       { hash: string }[]
     >`SELECT b1."hash" FROM "Block" b1 
@@ -185,6 +185,22 @@ export class PrismaBlockStorage
     });
   }
 
+  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+    const result = await this.getLatestBlockAndResult();
+    if (result !== undefined) {
+      if (result.result === undefined) {
+        throw new Error(
+          `Block result for block ${result.block.height.toString()} not found`
+        );
+      }
+      return {
+        block: result.block,
+        result: result.result,
+      };
+    }
+    return result;
+  }
+
   public async getNewBlocks(): Promise<BlockWithPreviousResult[]> {
     const blocks = await this.connection.prismaClient.block.findMany({
       where: {

packages/sdk/src/query/BlockStorageNetworkStateModule.ts

@@ -51,7 +51,7 @@ export class BlockStorageNetworkStateModule
    * with afterBundle() hooks executed
    */
   public async getStagedNetworkState(): Promise<NetworkState | undefined> {
-    const result = await this.unprovenQueue.getLatestBlock();
+    const result = await this.unprovenStorage.getLatestBlock();
     return result?.result.afterNetworkState;
   }
 

packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts

@@ -1,5 +1,5 @@
 import { inject } from "tsyringe";
-import { log, noop } from "@proto-kit/common";
+import { log } from "@proto-kit/common";
 import { ACTIONS_EMPTY_HASH } from "@proto-kit/protocol";
 import {
   MethodIdResolver,
@@ -18,7 +18,11 @@ import { BlockQueue } from "../../../storage/repositories/BlockStorage";
 import { PendingTransaction } from "../../../mempool/PendingTransaction";
 import { AsyncMerkleTreeStore } from "../../../state/async/AsyncMerkleTreeStore";
 import { AsyncStateService } from "../../../state/async/AsyncStateService";
-import { Block, BlockWithResult } from "../../../storage/model/Block";
+import {
+  Block,
+  BlockResult,
+  BlockWithResult,
+} from "../../../storage/model/Block";
 import { CachedStateService } from "../../../state/state/CachedStateService";
 import { MessageStorage } from "../../../storage/repositories/MessageStorage";
 
@@ -99,7 +103,23 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
     }
   }
 
-  public async tryProduceBlock(): Promise<BlockWithResult | undefined> {
+  public async generateMetadata(block: Block): Promise<BlockResult> {
+    const { result, blockHashTreeStore, treeStore } =
+      await this.executionService.generateMetadataForNextBlock(
+        block,
+        this.unprovenMerkleStore,
+        this.blockTreeStore
+      );
+
+    await blockHashTreeStore.mergeIntoParent();
+    await treeStore.mergeIntoParent();
+
+    await this.blockQueue.pushResult(result);
+
+    return result;
+  }
+
+  public async tryProduceBlock(): Promise<Block | undefined> {
     if (!this.productionInProgress) {
       try {
         const block = await this.produceBlock();
@@ -118,20 +138,7 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
         );
         this.prettyPrintBlockContents(block);
 
-        // Generate metadata for next block
-
-        // TODO: make async of production in the future
-        const result = await this.executionService.generateMetadataForNextBlock(
-          block,
-          this.unprovenMerkleStore,
-          this.blockTreeStore,
-          true
-        );
-
-        return {
-          block,
-          result,
-        };
+        return block;
       } catch (error: unknown) {
         if (error instanceof Error) {
           throw error;
@@ -151,19 +158,31 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
   }> {
     const txs = await this.mempool.getTxs(this.maximumBlockSize());
 
-    const parentBlock = await this.blockQueue.getLatestBlock();
+    const parentBlock = await this.blockQueue.getLatestBlockAndResult();
+
+    let metadata: BlockWithResult;
 
     if (parentBlock === undefined) {
       log.debug(
         "No block metadata given, assuming first block, generating genesis metadata"
       );
+      metadata = BlockWithResult.createEmpty();
+    } else if (parentBlock.result === undefined) {
+      throw new Error(
+        `Metadata for block at height ${parentBlock.block.height.toString()} not available`
+      );
+    } else {
+      metadata = {
+        block: parentBlock.block,
+        // By reconstructing this object, typescript correctly infers the result to be defined
+        result: parentBlock.result,
+      };
     }
 
     const messages = await this.messageStorage.getMessages(
       parentBlock?.block.toMessagesHash.toString() ??
         ACTIONS_EMPTY_HASH.toString()
     );
-    const metadata = parentBlock ?? BlockWithResult.createEmpty();
 
     log.debug(
       `Block collected, ${txs.length} txs, ${messages.length} messages`
@@ -196,14 +215,29 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
       this.allowEmptyBlock()
     );
 
-    await cachedStateService.mergeIntoParent();
+    if (block !== undefined) {
+      await cachedStateService.mergeIntoParent();
+
+      await this.blockQueue.pushBlock(block);
+    }
 
     this.productionInProgress = false;
 
     return block;
   }
 
   public async start() {
-    noop();
+    // Check if metadata height is behind block production.
+    // This can happen when the sequencer crashes after a block has been produced
+    // but before the metadata generation has finished
+    const latestBlock = await this.blockQueue.getLatestBlockAndResult();
+    // eslint-disable-next-line sonarjs/no-collapsible-if
+    if (latestBlock !== undefined) {
+      if (latestBlock.result === undefined) {
+        await this.generateMetadata(latestBlock.block);
+      }
+      // Here, the metadata has been computed already
+    }
+    // If we reach here, its a genesis startup, no blocks exist yet
   }
 }

packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts

@@ -323,9 +323,12 @@ export class TransactionExecutionService {
   public async generateMetadataForNextBlock(
     block: Block,
     merkleTreeStore: AsyncMerkleTreeStore,
-    blockHashTreeStore: AsyncMerkleTreeStore,
-    modifyTreeStore = true
-  ): Promise<BlockResult> {
+    blockHashTreeStore: AsyncMerkleTreeStore
+  ): Promise<{
+    result: BlockResult;
+    treeStore: CachedMerkleTreeStore;
+    blockHashTreeStore: CachedMerkleTreeStore;
+  }> {
     // Flatten diff list into a single diff by applying them over each other
     const combinedDiff = block.transactions
       .map((tx) => {
@@ -403,22 +406,21 @@ export class TransactionExecutionService {
     );
     const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt());
     const newBlockHashRoot = blockHashTree.getRoot();
-    await blockHashInMemoryStore.mergeIntoParent();
-
-    if (modifyTreeStore) {
-      await inMemoryStore.mergeIntoParent();
-    }
 
     return {
-      afterNetworkState: resultingNetworkState,
-      stateRoot: stateRoot.toBigInt(),
-      blockHashRoot: newBlockHashRoot.toBigInt(),
-      blockHashWitness,
-
-      blockStateTransitions: reducedStateTransitions.map((st) =>
-        UntypedStateTransition.fromStateTransition(st)
-      ),
-      blockHash: block.hash.toBigInt(),
+      result: {
+        afterNetworkState: resultingNetworkState,
+        stateRoot: stateRoot.toBigInt(),
+        blockHashRoot: newBlockHashRoot.toBigInt(),
+        blockHashWitness,
+
+        blockStateTransitions: reducedStateTransitions.map((st) =>
+          UntypedStateTransition.fromStateTransition(st)
+        ),
+        blockHash: block.hash.toBigInt(),
+      },
+      treeStore: inMemoryStore,
+      blockHashTreeStore: blockHashInMemoryStore,
     };
   }
 

packages/sequencer/src/protocol/production/trigger/BlockTrigger.ts

@@ -65,28 +65,29 @@ export class BlockTriggerBase<
     return undefined;
   }
 
-  protected async produceBlockWithResult(
-    enqueueInSettlementQueue: boolean
-  ): Promise<BlockWithResult | undefined> {
+  protected async produceBlockWithResult(): Promise<
+    BlockWithResult | undefined
+  > {
     const block = await this.blockProducerModule.tryProduceBlock();
+    if (block) {
+      this.events.emit("block-produced", block);
 
-    if (block && enqueueInSettlementQueue) {
-      await this.blockQueue.pushBlock(block.block);
-      this.events.emit("block-produced", block.block);
+      const result = await this.blockProducerModule.generateMetadata(block);
 
-      await this.blockQueue.pushResult(block.result);
-      this.events.emit("block-metadata-produced", block);
-    }
+      const blockWithMetadata = {
+        block,
+        result,
+      };
+
+      this.events.emit("block-metadata-produced", blockWithMetadata);
 
-    return block;
+      return blockWithMetadata;
+    }
+    return undefined;
   }
 
-  protected async produceBlock(
-    enqueueInSettlementQueue: boolean
-  ): Promise<Block | undefined> {
-    const blockWithResult = await this.produceBlockWithResult(
-      enqueueInSettlementQueue
-    );
+  protected async produceBlock(): Promise<Block | undefined> {
+    const blockWithResult = await this.produceBlockWithResult();
 
     return blockWithResult?.block;
   }

packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts

@@ -62,15 +62,11 @@ export class ManualBlockTrigger
     return await super.settle(batch);
   }
 
-  public async produceBlock(
-    enqueueInSettlementQueue: boolean = true
-  ): Promise<Block | undefined> {
-    return await super.produceBlock(enqueueInSettlementQueue);
+  public async produceBlock(): Promise<Block | undefined> {
+    return await super.produceBlock();
   }
 
-  public async produceBlockWithResult(
-    enqueueInSettlementQueue: boolean = true
-  ): Promise<BlockWithResult | undefined> {
-    return await super.produceBlockWithResult(enqueueInSettlementQueue);
+  public async produceBlockWithResult(): Promise<BlockWithResult | undefined> {
+    return await super.produceBlockWithResult();
   }
 }

packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts

@@ -129,7 +129,7 @@ export class TimedBlockTrigger
     // Produce a block if either produceEmptyBlocks is true or we have more
     // than 1 tx in mempool
     if (mempoolTxs.length > 0 || (this.config.produceEmptyBlocks ?? true)) {
-      await this.produceBlock(true);
+      await this.produceBlock();
     }
   }
 

packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts

@@ -5,7 +5,12 @@ import {
   BlockQueue,
   BlockStorage,
 } from "../repositories/BlockStorage";
-import type { Block, BlockResult, BlockWithResult } from "../model/Block";
+import type {
+  Block,
+  BlockResult,
+  BlockWithMaybeResult,
+  BlockWithResult,
+} from "../model/Block";
 import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule";
 import { BatchStorage } from "../repositories/BatchStorage";
 
@@ -29,10 +34,12 @@ export class InMemoryBlockStorage
     return this.blocks.length;
   }
 
-  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+  public async getLatestBlockAndResult(): Promise<
+    BlockWithMaybeResult | undefined
+  > {
     const currentHeight = await this.getCurrentBlockHeight();
     const block = await this.getBlockAt(currentHeight - 1);
-    const result = this.results[currentHeight - 1];
+    const result: BlockResult | undefined = this.results[currentHeight - 1];
     if (block === undefined) {
       return undefined;
     }
@@ -42,6 +49,22 @@ export class InMemoryBlockStorage
     };
   }
 
+  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+    const result = await this.getLatestBlockAndResult();
+    if (result !== undefined) {
+      if (result.result === undefined) {
+        throw new Error(
+          `Block result for block ${result.block.height.toString()} not found`
+        );
+      }
+      return {
+        block: result.block,
+        result: result.result,
+      };
+    }
+    return result;
+  }
+
   public async getNewBlocks(): Promise<BlockWithPreviousResult[]> {
     const latestBatch = await this.batchStorage.getLatestBatch();