feat: de-duplicate payloads from persisted beacon blocks

.github/workflows/test-sim-merge.yml

@@ -58,18 +58,30 @@ jobs:
       - name: Pull Nethermind
         run: docker pull $NETHERMIND_IMAGE
 
-      - name: Pull mergemock
-        run: docker pull $MERGEMOCK_IMAGE
-
-      - name: Test Lodestar <> mergemock relay
-        run: yarn test:sim:mergemock
+      - name: Test Lodestar <> Nethermind interop
+        run: yarn test:sim:merge-interop
         working-directory: packages/beacon-node
         env:
-          EL_BINARY_DIR: ${{ env.MERGEMOCK_IMAGE }}
-          EL_SCRIPT_DIR: mergemock
-          LODESTAR_PRESET: mainnet
+          EL_BINARY_DIR: ${{ env.NETHERMIND_IMAGE }}
+          EL_SCRIPT_DIR: netherminddocker
           ENGINE_PORT: 8551
-          ETH_PORT: 8661
+          ETH_PORT: 8545
+
+      # This container is pre-shanghai and does not support enginer_getPayloadBodyV2
+      # for blinding/unblinding. Re-enable when we have a newer build.
+      # 
+      # - name: Pull mergemock
+      #   run: docker pull $MERGEMOCK_IMAGE
+
+      # - name: Test Lodestar <> mergemock relay
+      #   run: yarn test:sim:mergemock
+      #   working-directory: packages/beacon-node
+      #   env:
+      #     EL_BINARY_DIR: ${{ env.MERGEMOCK_IMAGE }}
+      #     EL_SCRIPT_DIR: mergemock
+      #     LODESTAR_PRESET: mainnet
+      #     ENGINE_PORT: 8551
+      #     ETH_PORT: 8661
 
       - name: Upload debug log test files
         if: ${{ always() }}

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -281,6 +281,7 @@ export function getBeaconBlockApi({
 
   return {
     async getBlockHeaders({slot, parentRoot}) {
+      // TODO: (matthewkeil) Make this code BlindedOrFull block aware
       // TODO - SLOW CODE: This code seems like it could be improved
 
       // If one block in the response contains an optimistic block, mark the entire response as optimistic

packages/beacon-node/src/api/impl/beacon/blocks/utils.ts

@@ -1,23 +1,38 @@
 import {routes} from "@lodestar/api";
-import {blockToHeader} from "@lodestar/state-transition";
 import {ChainForkConfig} from "@lodestar/config";
 import {SignedBeaconBlock} from "@lodestar/types";
 import {GENESIS_SLOT} from "../../../../constants/index.js";
 import {ApiError, ValidationError} from "../../errors.js";
 import {IBeaconChain} from "../../../../chain/interface.js";
 import {rootHexRegex} from "../../../../eth1/provider/utils.js";
+import {isBlinded} from "../../../../util/fullOrBlindedBlock.js";
 
 export function toBeaconHeaderResponse(
   config: ChainForkConfig,
   block: SignedBeaconBlock,
   canonical = false
 ): routes.beacon.BlockHeaderResponse {
+  // need to have ts-ignore below to pull type here so it only happens once and
+  // gets used twice
+  const types = isBlinded(block)
+    ? config.getExecutionForkTypes(block.message.slot)
+    : config.getForkTypes(block.message.slot);
   return {
-    root: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message),
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    root: types.BeaconBlock.hashTreeRoot(block.message),
     canonical,
     header: {
-      message: blockToHeader(config, block.message),
       signature: block.signature,
+      message: {
+        stateRoot: block.message.stateRoot,
+        proposerIndex: block.message.proposerIndex,
+        slot: block.message.slot,
+        parentRoot: block.message.parentRoot,
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        bodyRoot: types.BeaconBlockBody.hashTreeRoot(block.message.body),
+      },
     },
   };
 }

packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,5 +1,6 @@
 import {toHex} from "@lodestar/utils";
 import {BeaconChain} from "../chain.js";
+import {blindedOrFullBlockToBlinded} from "../../util/fullOrBlindedBlock.js";
 import {BlockInput, BlockInputType} from "./types.js";
 
 /**
@@ -13,17 +14,11 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI
   const fnPromises: Promise<void>[] = [];
 
   for (const blockInput of blocksInput) {
-    const {block, blockBytes} = blockInput;
+    const {block} = blockInput;
     const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
     const blockRootHex = toHex(blockRoot);
-    if (blockBytes) {
-      // skip serializing data if we already have it
-      this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-      fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
-    } else {
-      this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-      fnPromises.push(this.db.block.add(block));
-    }
+    this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
+    fnPromises.push(this.db.block.add(blindedOrFullBlockToBlinded(this.config, block)));
     this.logger.debug("Persist block to hot DB", {
       slot: block.message.slot,
       root: blockRootHex,

packages/beacon-node/src/chain/chain.ts

@@ -29,6 +29,7 @@ import {
   isBlindedBeaconBlock,
   BeaconBlock,
   SignedBeaconBlock,
+  FullOrBlindedSignedBeaconBlock,
   ExecutionPayload,
   BlindedBeaconBlock,
   BlindedBeaconBlockBody,
@@ -47,6 +48,14 @@ import {Clock, ClockEvent, IClock} from "../util/clock.js";
 import {ensureDir, writeIfNotExist} from "../util/file.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
 import {BufferPool} from "../util/bufferPool.js";
+import {
+  blindedOrFullBlockToFull,
+  deserializeFullOrBlindedSignedBeaconBlock,
+  getEth1BlockHashFromSerializedBlock,
+  serializeFullOrBlindedSignedBeaconBlock,
+} from "../util/fullOrBlindedBlock.js";
+import {ExecutionPayloadBody} from "../execution/engine/types.js";
+import {Eth1Error, Eth1ErrorCode} from "../eth1/errors.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
 import {ChainEventEmitter, ChainEvent} from "./emitter.js";
 import {
@@ -510,7 +519,11 @@ export class BeaconChain implements IBeaconChain {
       if (block) {
         const data = await this.db.block.get(fromHexString(block.blockRoot));
         if (data) {
-          return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
+          return {
+            block: await this.blindedOrFullBlockToFull(data),
+            executionOptimistic: isOptimisticBlock(block),
+            finalized: false,
+          };
         }
       }
       // A non-finalized slot expected to be found in the hot db, could be archived during
@@ -519,7 +532,7 @@ export class BeaconChain implements IBeaconChain {
     }
 
     const data = await this.db.blockArchive.get(slot);
-    return data && {block: data, executionOptimistic: false, finalized: true};
+    return data && {block: await this.blindedOrFullBlockToFull(data), executionOptimistic: false, finalized: true};
   }
 
   async getBlockByRoot(
@@ -529,7 +542,11 @@ export class BeaconChain implements IBeaconChain {
     if (block) {
       const data = await this.db.block.get(fromHexString(root));
       if (data) {
-        return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
+        return {
+          block: await this.blindedOrFullBlockToFull(data),
+          executionOptimistic: isOptimisticBlock(block),
+          finalized: false,
+        };
       }
       // If block is not found in hot db, try cold db since there could be an archive cycle happening
       // TODO: Add a lock to the archiver to have deterministic behavior on where are blocks
@@ -574,6 +591,28 @@ export class BeaconChain implements IBeaconChain {
     return this.produceBlockWrapper<BlockType.Blinded>(BlockType.Blinded, blockAttributes);
   }
 
+  async blindedOrFullBlockToFull(block: FullOrBlindedSignedBeaconBlock): Promise<SignedBeaconBlock> {
+    const info = this.config.getForkInfo(block.message.slot);
+    return blindedOrFullBlockToFull(
+      this.config,
+      info.seq,
+      block,
+      await this.getTransactionsAndWithdrawals(info.seq, toHexString(block.message.body.eth1Data.blockHash))
+    );
+  }
+
+  async blindedOrFullBlockToFullBytes(forkSeq: ForkSeq, block: Uint8Array): Promise<Uint8Array> {
+    return serializeFullOrBlindedSignedBeaconBlock(
+      this.config,
+      blindedOrFullBlockToFull(
+        this.config,
+        forkSeq,
+        deserializeFullOrBlindedSignedBeaconBlock(this.config, block),
+        await this.getTransactionsAndWithdrawals(forkSeq, toHexString(getEth1BlockHashFromSerializedBlock(block)))
+      )
+    );
+  }
+
   async produceBlockWrapper<T extends BlockType>(
     blockType: T,
     {
@@ -803,6 +842,23 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
+  private async getTransactionsAndWithdrawals(
+    forkSeq: ForkSeq,
+    blockHash: string
+  ): Promise<Partial<ExecutionPayloadBody>> {
+    if (forkSeq < ForkSeq.bellatrix) {
+      return {};
+    }
+    const [payload] = await this.executionEngine.getPayloadBodiesByHash([blockHash]);
+    if (!payload) {
+      throw new Eth1Error(
+        {code: Eth1ErrorCode.INVALID_PAYLOAD_BODY, blockHash},
+        "payload body not found by eth1 engine"
+      );
+    }
+    return payload;
+  }
+
   /**
    * Regenerate state for attestation verification, this does not happen with default chain option of maxSkipSlots = 32 .
    * However, need to handle just in case. Lodestar doesn't support multiple regen state requests for attestation verification

packages/beacon-node/src/chain/interface.ts

@@ -14,6 +14,7 @@ import {
   BeaconBlock,
   ExecutionPayload,
   SignedBeaconBlock,
+  FullOrBlindedSignedBeaconBlock,
   BlindedBeaconBlock,
 } from "@lodestar/types";
 import {
@@ -26,6 +27,7 @@ import {
 import {BeaconConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
 import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkSeq} from "@lodestar/params";
 import {IEth1ForBlockProduction} from "../eth1/index.js";
 import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
@@ -187,6 +189,9 @@ export interface IBeaconChain {
     consensusBlockValue: Wei;
   }>;
 
+  blindedOrFullBlockToFull(block: FullOrBlindedSignedBeaconBlock): Promise<SignedBeaconBlock>;
+  blindedOrFullBlockToFullBytes(forkSeq: ForkSeq, block: Uint8Array): Promise<Uint8Array>;
+
   /** Process a block until complete */
   processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise<void>;
   /** Process a chain of blocks until complete */

packages/beacon-node/src/db/repositories/block.ts

@@ -1,6 +1,11 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, Repository} from "@lodestar/db";
-import {SignedBeaconBlock, ssz} from "@lodestar/types";
+import {ssz, FullOrBlindedSignedBeaconBlock} from "@lodestar/types";
+import {blindedOrFullBlockHashTreeRoot} from "@lodestar/state-transition";
+import {
+  deserializeFullOrBlindedSignedBeaconBlock,
+  serializeFullOrBlindedSignedBeaconBlock,
+} from "../../util/fullOrBlindedBlock.js";
 import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
@@ -9,7 +14,7 @@ import {Bucket, getBucketNameByValue} from "../buckets.js";
  *
  * Used to store unfinalized blocks
  */
-export class BlockRepository extends Repository<Uint8Array, SignedBeaconBlock> {
+export class BlockRepository extends Repository<Uint8Array, FullOrBlindedSignedBeaconBlock> {
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_block;
     const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
@@ -19,15 +24,15 @@ export class BlockRepository extends Repository<Uint8Array, SignedBeaconBlock> {
   /**
    * Id is hashTreeRoot of unsigned BeaconBlock
    */
-  getId(value: SignedBeaconBlock): Uint8Array {
-    return this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+  getId(value: FullOrBlindedSignedBeaconBlock): Uint8Array {
+    return blindedOrFullBlockHashTreeRoot(this.config, value.message);
   }
 
-  encodeValue(value: SignedBeaconBlock): Buffer {
-    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Buffer;
+  encodeValue(value: FullOrBlindedSignedBeaconBlock): Buffer {
+    return serializeFullOrBlindedSignedBeaconBlock(this.config, value) as Buffer;
   }
 
-  decodeValue(data: Buffer): SignedBeaconBlock {
-    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  decodeValue(data: Buffer): FullOrBlindedSignedBeaconBlock {
+    return deserializeFullOrBlindedSignedBeaconBlock(this.config, data);
   }
 }