From ff6ed91855c375120ff79354e3d43fcd94688984 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 15 Oct 2024 10:18:11 +0700 Subject: [PATCH 1/3] feat: improve type.hashTreeRoot() using batch --- packages/as-sha256/src/index.ts | 1 + packages/ssz/src/type/abstract.ts | 5 + packages/ssz/src/type/arrayComposite.ts | 26 +++-- packages/ssz/src/type/basic.ts | 13 ++- packages/ssz/src/type/bitArray.ts | 13 ++- packages/ssz/src/type/bitList.ts | 30 +++++- packages/ssz/src/type/byteArray.ts | 31 +++++- packages/ssz/src/type/byteList.ts | 31 +++++- packages/ssz/src/type/composite.ts | 35 ++++++- packages/ssz/src/type/container.ts | 13 +-- packages/ssz/src/type/containerNodeStruct.ts | 1 - packages/ssz/src/type/listBasic.ts | 61 +++++++++--- packages/ssz/src/type/listComposite.ts | 54 ++++++++--- packages/ssz/src/type/optional.ts | 34 +++++-- packages/ssz/src/type/profile.ts | 37 +++++--- packages/ssz/src/type/stableContainer.ts | 79 +++++++++------- packages/ssz/src/type/uint.ts | 6 ++ packages/ssz/src/type/union.ts | 29 ++++-- packages/ssz/src/type/vectorBasic.ts | 16 +++- packages/ssz/src/type/vectorComposite.ts | 10 +- packages/ssz/src/util/merkleize.ts | 41 ++++---- .../ssz/test/perf/eth2/beaconBlock.test.ts | 94 +++++++++++++++++++ packages/ssz/test/perf/merkleize.test.ts | 20 +++- packages/ssz/test/spec/runValidTest.ts | 9 +- packages/ssz/test/unit/merkleize.test.ts | 35 ++++++- 25 files changed, 565 insertions(+), 159 deletions(-) create mode 100644 packages/ssz/test/perf/eth2/beaconBlock.test.ts diff --git a/packages/as-sha256/src/index.ts b/packages/as-sha256/src/index.ts index 11f44cc3..46d60789 100644 --- a/packages/as-sha256/src/index.ts +++ b/packages/as-sha256/src/index.ts @@ -3,6 +3,7 @@ import {newInstance} from "./wasm"; import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject"; import SHA256 from "./sha256"; export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256}; +export {allocUnsafe}; const ctx = newInstance(); const wasmInputValue = ctx.input.value; diff --git a/packages/ssz/src/type/abstract.ts b/packages/ssz/src/type/abstract.ts index b96b7355..792ca077 100644 --- a/packages/ssz/src/type/abstract.ts +++ b/packages/ssz/src/type/abstract.ts @@ -145,6 +145,11 @@ export abstract class Type { */ abstract hashTreeRoot(value: V): Uint8Array; + /** + * Same to hashTreeRoot() but here we write result to output. + */ + abstract hashTreeRootInto(value: V, output: Uint8Array, offset: number): void; + // JSON support /** Parse JSON representation of a type to value */ diff --git a/packages/ssz/src/type/arrayComposite.ts b/packages/ssz/src/type/arrayComposite.ts index d3b0a8fb..986b0e0a 100644 --- a/packages/ssz/src/type/arrayComposite.ts +++ b/packages/ssz/src/type/arrayComposite.ts @@ -211,21 +211,29 @@ export function tree_deserializeFromBytesArrayComposite>( +export function value_getChunkBytesArrayComposite>( elementType: ElementType, length: number, - value: ValueOf[] -): Uint8Array[] { - const roots = new Array(length); + value: ValueOf[], + chunkBytesBuffer: Uint8Array +): Uint8Array { + const isOddChunk = length % 2 === 1; + const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32; + if (chunkBytesLen > chunkBytesBuffer.length) { + throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`); + } + const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen); for (let i = 0; i < length; i++) { - roots[i] = elementType.hashTreeRoot(value[i]); + elementType.hashTreeRootInto(value[i], chunkBytes, i * 32); + } + + if (isOddChunk) { + // similar to append zeroHash(0) + chunkBytes.subarray(length * 32, chunkBytesLen).fill(0); } - return roots; + return chunkBytes; } function readOffsetsArrayComposite( diff --git a/packages/ssz/src/type/basic.ts b/packages/ssz/src/type/basic.ts index 0260ea49..920c6d97 100644 --- a/packages/ssz/src/type/basic.ts +++ b/packages/ssz/src/type/basic.ts @@ -30,11 +30,18 @@ export abstract class BasicType extends Type { } hashTreeRoot(value: V): Uint8Array { - // TODO: Optimize - const uint8Array = new Uint8Array(32); + // cannot use allocUnsafe() here because hashTreeRootInto() may not fill the whole 32 bytes + const root = new Uint8Array(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: V, output: Uint8Array, offset: number): void { + const uint8Array = output.subarray(offset, offset + 32); + // output could have preallocated data, some types may not fill the whole 32 bytes + uint8Array.fill(0); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); this.value_serializeToBytes({uint8Array, dataView}, 0, value); - return uint8Array; } clone(value: V): V { diff --git a/packages/ssz/src/type/bitArray.ts b/packages/ssz/src/type/bitArray.ts index 5071550c..d485de27 100644 --- a/packages/ssz/src/type/bitArray.ts +++ b/packages/ssz/src/type/bitArray.ts @@ -1,10 +1,10 @@ import {concatGindices, Gindex, Node, toGindex, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray"; -import {splitIntoRootChunks} from "../util/merkleize"; import {CompositeType, LENGTH_GINDEX} from "./composite"; import {BitArray} from "../value/bitArray"; import {BitArrayTreeView} from "../view/bitArray"; import {BitArrayTreeViewDU} from "../viewDU/bitArray"; +import {getChunkBytes} from "./byteArray"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -40,8 +40,15 @@ export abstract class BitArrayType extends CompositeType this.chunkBytesBuffer.length) { + const chunkCount = Math.ceil(value.bitLen / 8 / 32); + const chunkBytes = chunkCount * 32; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + } + return getChunkBytes(value.uint8Array, this.chunkBytesBuffer); } // Proofs diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index 0d8268b2..ba1c419a 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -1,5 +1,12 @@ -import {getNodesAtDepth, Node, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree"; -import {mixInLength, maxChunksToDepth} from "../util/merkleize"; +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + getNodesAtDepth, + merkleizeInto, + Node, + packedNodeRootsToBytes, + packedRootsBytesToNode, +} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ByteViews} from "./composite"; @@ -29,6 +36,12 @@ export class BitListType extends BitArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly limitBits: number, opts?: BitListOptions) { super(); @@ -101,7 +114,18 @@ export class BitListType extends BitArrayType { // Merkleization: inherited from BitArrayType hashTreeRoot(value: BitArray): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.bitLen); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/byteArray.ts b/packages/ssz/src/type/byteArray.ts index 202f1b24..78e6ae30 100644 --- a/packages/ssz/src/type/byteArray.ts +++ b/packages/ssz/src/type/byteArray.ts @@ -8,7 +8,6 @@ import { getHashComputations, } from "@chainsafe/persistent-merkle-tree"; import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray"; -import {splitIntoRootChunks} from "../util/merkleize"; import {ByteViews} from "./abstract"; import {CompositeType, LENGTH_GINDEX} from "./composite"; @@ -82,10 +81,23 @@ export abstract class ByteArrayType extends CompositeType this.chunkBytesBuffer.length) { + const chunkCount = Math.ceil(value.length / 32); + const chunkBytes = chunkCount * 32; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + } + return getChunkBytes(value, this.chunkBytesBuffer); } // Proofs @@ -149,3 +161,16 @@ export abstract class ByteArrayType extends CompositeType merkleBytesBuffer.length) { + throw new Error(`data length ${data.length} exceeds merkleBytesBuffer length ${merkleBytesBuffer.length}`); + } + + merkleBytesBuffer.set(data); + const valueLen = data.length; + const chunkByteLen = Math.ceil(valueLen / 64) * 64; + // all padding bytes must be zero, this is similar to set zeroHash(0) + merkleBytesBuffer.subarray(valueLen, chunkByteLen).fill(0); + return merkleBytesBuffer.subarray(0, chunkByteLen); +} diff --git a/packages/ssz/src/type/byteList.ts b/packages/ssz/src/type/byteList.ts index 6f12fff7..53c46d39 100644 --- a/packages/ssz/src/type/byteList.ts +++ b/packages/ssz/src/type/byteList.ts @@ -1,11 +1,17 @@ -import {getNodesAtDepth, Node, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree"; -import {mixInLength, maxChunksToDepth} from "../util/merkleize"; +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + getNodesAtDepth, + Node, + packedNodeRootsToBytes, + packedRootsBytesToNode, + merkleizeInto, +} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {addLengthNode, getChunksNodeFromRootNode, getLengthFromRootNode} from "./arrayBasic"; import {ByteViews} from "./composite"; import {ByteArrayType, ByteArray} from "./byteArray"; - /* eslint-disable @typescript-eslint/member-ordering */ export interface ByteListOptions { @@ -34,6 +40,12 @@ export class ByteListType extends ByteArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly limitBytes: number, opts?: ByteListOptions) { super(); @@ -89,7 +101,18 @@ export class ByteListType extends ByteArrayType { // Merkleization: inherited from ByteArrayType hashTreeRoot(value: ByteArray): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/composite.ts b/packages/ssz/src/type/composite.ts index c403c385..9f33ca5e 100644 --- a/packages/ssz/src/type/composite.ts +++ b/packages/ssz/src/type/composite.ts @@ -1,3 +1,4 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; import { concatGindices, createProof, @@ -7,10 +8,11 @@ import { Proof, ProofType, Tree, + merkleizeInto, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; import {byteArrayEquals} from "../util/byteArray"; -import {merkleize, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; +import {cacheRoot, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {treePostProcessFromProofNode} from "../util/proof/treePostProcessFromProofNode"; import {Type, ByteViews, JsonPath, JsonPathProp} from "./abstract"; export {ByteViews}; @@ -59,6 +61,7 @@ export abstract class CompositeType extends Type { * Required for ContainerNodeStruct to ensure no dangerous types are constructed. */ abstract readonly isViewMutable: boolean; + protected chunkBytesBuffer = new Uint8Array(0); constructor( /** @@ -216,13 +219,30 @@ export abstract class CompositeType extends Type { } } - const root = merkleize(this.getRoots(value), this.maxChunkCount); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + + return root; + } + + hashTreeRootInto(value: V, output: Uint8Array, offset: number, safeCache = false): void { + // Return cached mutable root if any if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, output, offset); + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } // For debugging and testing this feature @@ -236,7 +256,12 @@ export abstract class CompositeType extends Type { // and feed those numbers directly to the hasher input with a DataView // - The return of the hasher should be customizable too, to reduce conversions from Uint8Array // to hashObject and back. - protected abstract getRoots(value: V): Uint8Array[]; + + /** + * Get merkle bytes of each value, the returned Uint8Array should be multiple of 64 bytes. + * If chunk count is not even, need to append zeroHash(0) + */ + protected abstract getChunkBytes(value: V): Uint8Array; // Proofs API diff --git a/packages/ssz/src/type/container.ts b/packages/ssz/src/type/container.ts index daa1911d..1ed46a89 100644 --- a/packages/ssz/src/type/container.ts +++ b/packages/ssz/src/type/container.ts @@ -130,6 +130,9 @@ export class ContainerType>> extends // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + const fieldBytes = this.fieldsEntries.length * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -272,15 +275,13 @@ export class ContainerType>> extends // Merkleization - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.fieldsEntries.length); - + protected getChunkBytes(struct: ValueOfFields): Uint8Array { for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType} = this.fieldsEntries[i]; - roots[i] = fieldType.hashTreeRoot(struct[fieldName]); + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); } - - return roots; + // remaining bytes are zeroed as we never write them + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/containerNodeStruct.ts b/packages/ssz/src/type/containerNodeStruct.ts index 8cefa381..76147f12 100644 --- a/packages/ssz/src/type/containerNodeStruct.ts +++ b/packages/ssz/src/type/containerNodeStruct.ts @@ -106,7 +106,6 @@ export class ContainerNodeStructType return new BranchNodeStruct(this.valueToTree.bind(this), value); } - // TODO: Optimize conversion private valueToTree(value: ValueOfFields): Node { const uint8Array = new Uint8Array(this.value_serializedSize(value)); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); diff --git a/packages/ssz/src/type/listBasic.ts b/packages/ssz/src/type/listBasic.ts index c9e397e6..4dd63f08 100644 --- a/packages/ssz/src/type/listBasic.ts +++ b/packages/ssz/src/type/listBasic.ts @@ -1,4 +1,4 @@ -import {LeafNode, Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; +import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; import {ValueOf} from "./abstract"; import {BasicType} from "./basic"; import {ByteViews} from "./composite"; @@ -10,19 +10,14 @@ import { addLengthNode, setChunksNode, } from "./arrayBasic"; -import { - mixInLength, - maxChunksToDepth, - splitIntoRootChunks, - symbolCachedPermanentRoot, - ValueWithCachedPermanentRoot, -} from "../util/merkleize"; +import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ArrayBasicType} from "../view/arrayBasic"; import {ListBasicTreeView} from "../view/listBasic"; import {ListBasicTreeViewDU} from "../viewDU/listBasic"; import {ArrayType} from "./array"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -52,6 +47,12 @@ export class ListBasicType> readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly defaultLen = 0; constructor(readonly elementType: ElementType, readonly limit: number, opts?: ListBasicOpts) { @@ -174,20 +175,52 @@ export class ListBasicType> } } - const root = mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } - protected getRoots(value: ValueOf[]): Uint8Array[] { - const uint8Array = new Uint8Array(this.value_serializedSize(value)); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const byteLen = this.value_serializedSize(value); + const chunkByteLen = Math.ceil(byteLen / 64) * 64; + // reallocate this.verkleBytes if needed + if (byteLen > this.chunkBytesBuffer.length) { + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = new Uint8Array(chunkByteLen); + } + const chunkBytes = this.chunkBytesBuffer.subarray(0, chunkByteLen); + const uint8Array = chunkBytes.subarray(0, byteLen); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value); - return splitIntoRootChunks(uint8Array); + + // all padding bytes must be zero, this is similar to set zeroHash(0) + this.chunkBytesBuffer.subarray(byteLen, chunkByteLen).fill(0); + return chunkBytes; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/listComposite.ts b/packages/ssz/src/type/listComposite.ts index dad8e77c..821b9504 100644 --- a/packages/ssz/src/type/listComposite.ts +++ b/packages/ssz/src/type/listComposite.ts @@ -1,10 +1,5 @@ -import {Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; -import { - mixInLength, - maxChunksToDepth, - symbolCachedPermanentRoot, - ValueWithCachedPermanentRoot, -} from "../util/merkleize"; +import {HashComputationLevel, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ValueOf, ByteViews} from "./abstract"; @@ -17,13 +12,14 @@ import { tree_serializedSizeArrayComposite, tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, - value_getRootsArrayComposite, maxSizeArrayComposite, + value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType} from "../view/arrayComposite"; import {ListCompositeTreeView} from "../view/listComposite"; import {ListCompositeTreeViewDU} from "../viewDU/listComposite"; import {ArrayType} from "./array"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -56,6 +52,12 @@ export class ListCompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly defaultLen = 0; constructor(readonly elementType: ElementType, readonly limit: number, opts?: ListCompositeOpts) { @@ -180,17 +182,43 @@ export class ListCompositeType< } } - const root = mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } - protected getRoots(value: ValueOf[]): Uint8Array[] { - return value_getRootsArrayComposite(this.elementType, value.length, value); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const byteLen = value.length * 32; + const chunkByteLen = this.chunkBytesBuffer.byteLength; + if (byteLen > chunkByteLen) { + this.chunkBytesBuffer = new Uint8Array(Math.ceil(byteLen / 64) * 64); + } + return value_getChunkBytesArrayComposite(this.elementType, value.length, value, this.chunkBytesBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 7c5f9baf..3d2925fb 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -1,18 +1,19 @@ import { concatGindices, Gindex, + merkleizeInto, Node, Tree, zeroNode, - HashComputationLevel, getHashComputations, + HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; -import {mixInLength} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ByteViews, JsonPath, JsonPathProp} from "./abstract"; import {CompositeType, isCompositeType} from "./composite"; import {addLengthNode, getLengthFromRootNode} from "./arrayBasic"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ export type NonOptionalType> = T extends OptionalType ? U : T; @@ -47,6 +48,12 @@ export class OptionalType> extends CompositeTy readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly elementType: ElementType, opts?: OptionalOpts) { super(); @@ -59,6 +66,7 @@ export class OptionalType> extends CompositeTy this.minSize = 0; // Max size includes prepended 0x01 byte this.maxSize = elementType.maxSize + 1; + this.chunkBytesBuffer = new Uint8Array(32); } static named>( @@ -171,13 +179,27 @@ export class OptionalType> extends CompositeTy // Merkleization hashTreeRoot(value: ValueOfType): Uint8Array { + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: ValueOfType, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); const selector = value === null ? 0 : 1; - return mixInLength(super.hashTreeRoot(value), selector); + this.mixInLengthBuffer.writeUIntLE(selector, 32, 6); + // one for hashTreeRoot(value), one for selector + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } - protected getRoots(value: ValueOfType): Uint8Array[] { - const valueRoot = value === null ? new Uint8Array(32) : this.elementType.hashTreeRoot(value); - return [valueRoot]; + protected getChunkBytes(value: ValueOfType): Uint8Array { + if (value === null) { + this.chunkBytesBuffer.fill(0); + } else { + this.elementType.hashTreeRootInto(value, this.chunkBytesBuffer, 0); + } + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts index f9469fe0..1ac440d7 100644 --- a/packages/ssz/src/type/profile.ts +++ b/packages/ssz/src/type/profile.ts @@ -6,12 +6,13 @@ import { Gindex, toGindex, concatGindices, + merkleizeInto, getNode, BranchNode, zeroHash, zeroNode, } from "@chainsafe/persistent-merkle-tree"; -import {ValueWithCachedPermanentRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; +import {ValueWithCachedPermanentRoot, cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ValueOf} from "./abstract"; @@ -87,6 +88,8 @@ export class ProfileType>> extends C protected readonly TreeView: ContainerTreeViewTypeConstructor; protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; private optionalFieldsCount: number; + // temporary root to avoid memory allocation + private tempRoot = new Uint8Array(32); constructor(readonly fields: Fields, activeFields: BitArray, readonly opts?: ProfileOptions) { super(); @@ -154,6 +157,9 @@ export class ProfileType>> extends C // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getProfileTreeViewClass?.(this) ?? getProfileTreeViewClass(this); this.TreeViewDU = opts?.getProfileTreeViewDUClass?.(this) ?? getProfileTreeViewDUClass(this); + const fieldBytes = this.activeFields.bitLen * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -361,37 +367,38 @@ export class ProfileType>> extends C } // Merkleization - hashTreeRoot(value: ValueOfFields): Uint8Array { + // hashTreeRoot is the same to parent as it call hashTreeRootInto() + hashTreeRootInto(value: ValueOfFields, output: Uint8Array, offset: number, safeCache = false): void { // Return cached mutable root if any if (this.cachePermanentRootStruct) { const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; if (cachedRoot) { - return cachedRoot; + output.set(cachedRoot, offset); + return; } } - const root = mixInActiveFields(super.hashTreeRoot(value), this.activeFields); + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + mixInActiveFields(this.tempRoot, this.activeFields, output, offset); if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } - - return root; } - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.activeFields.bitLen).fill(zeroHash(0)); - - // already asserted that # of active fields in bitvector === # of fields + protected getChunkBytes(struct: ValueOfFields): Uint8Array { + this.chunkBytesBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, chunkIndex, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - continue; + this.chunkBytesBuffer.set(zeroHash(0), chunkIndex * 32); + } else { + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, chunkIndex * 32); } - roots[chunkIndex] = fieldType.hashTreeRoot(struct[fieldName]); } - - return roots; + // remaining bytes are zeroed as we never write them + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index bf8b94fa..35415ad8 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -11,19 +11,13 @@ import { getNode, zeroNode, zeroHash, + merkleizeInto, countToDepth, getNodeH, setNode, setNodeWithFn, } from "@chainsafe/persistent-merkle-tree"; -import { - ValueWithCachedPermanentRoot, - hash64, - maxChunksToDepth, - merkleize, - splitIntoRootChunks, - symbolCachedPermanentRoot, -} from "../util/merkleize"; +import {ValueWithCachedPermanentRoot, cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {JsonPath, Type, ValueOf} from "./abstract"; @@ -99,6 +93,8 @@ export class StableContainerType>> e protected readonly TreeView: ContainerTreeViewTypeConstructor; protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; private padActiveFields: boolean[]; + // temporary root to avoid memory allocation + private tempRoot = new Uint8Array(32); constructor(fields: Fields, readonly maxFields: number, readonly opts?: StableContainerOptions) { super(); @@ -153,6 +149,9 @@ export class StableContainerType>> e // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + const fieldBytes = this.fieldsEntries.length * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -341,43 +340,43 @@ export class StableContainerType>> e } // Merkleization - hashTreeRoot(value: ValueOfFields): Uint8Array { + // hashTreeRoot is the same to parent as it call hashTreeRootInto() + hashTreeRootInto(value: ValueOfFields, output: Uint8Array, offset: number, safeCache = false): void { // Return cached mutable root if any if (this.cachePermanentRootStruct) { const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; if (cachedRoot) { - return cachedRoot; + output.set(cachedRoot, offset); + return; } } + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); // compute active field bitvector const activeFields = BitArray.fromBoolArray([ ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), ...this.padActiveFields, ]); - const root = mixInActiveFields(super.hashTreeRoot(value), activeFields); + mixInActiveFields(this.tempRoot, activeFields, output, offset); if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } - - return root; } - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.fieldsEntries.length); - + protected getChunkBytes(struct: ValueOfFields): Uint8Array { + this.chunkBytesBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - roots[i] = zeroHash(0); - continue; + this.chunkBytesBuffer.set(zeroHash(0), i * 32); + } else { + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); } - - roots[i] = fieldType.hashTreeRoot(struct[fieldName]); } - return roots; + return this.chunkBytesBuffer; } // Proofs @@ -751,12 +750,15 @@ export function getActiveFields(rootNode: Node, bitLen: number): BitArray { return new BitArray(activeFieldsBuf, bitLen); } +// This is a global buffer to avoid creating a new one for each call to getActiveFields +const singleChunkActiveFieldsBuf = new Uint8Array(32); + export function setActiveFields(rootNode: Node, activeFields: BitArray): Node { // fast path for depth 1, the bitvector fits in one chunk if (activeFields.bitLen <= 256) { - const activeFieldsBuf = new Uint8Array(32); - activeFieldsBuf.set(activeFields.uint8Array); - return new BranchNode(rootNode.left, LeafNode.fromRoot(activeFieldsBuf)); + singleChunkActiveFieldsBuf.fill(0); + singleChunkActiveFieldsBuf.set(activeFields.uint8Array); + return new BranchNode(rootNode.left, LeafNode.fromRoot(singleChunkActiveFieldsBuf)); } const activeFieldsChunkCount = Math.ceil(activeFields.bitLen / 256); @@ -815,15 +817,24 @@ export function setActiveField(rootNode: Node, bitLen: number, fieldIndex: numbe return new BranchNode(rootNode.left, newActiveFieldsNode); } -export function mixInActiveFields(root: Uint8Array, activeFields: BitArray): Uint8Array { +// This is a global buffer to avoid creating a new one for each call to getChunkBytes +const mixInActiveFieldsChunkBytes = new Uint8Array(64); +const activeFieldsSingleChunk = mixInActiveFieldsChunkBytes.subarray(32); + +export function mixInActiveFields(root: Uint8Array, activeFields: BitArray, output: Uint8Array, offset: number): void { // fast path for depth 1, the bitvector fits in one chunk + mixInActiveFieldsChunkBytes.set(root, 0); if (activeFields.bitLen <= 256) { - const activeFieldsChunk = new Uint8Array(32); - activeFieldsChunk.set(activeFields.uint8Array); - return hash64(root, activeFieldsChunk); - } - - const activeFieldsChunks = splitIntoRootChunks(activeFields.uint8Array); - const activeFieldsRoot = merkleize(activeFieldsChunks, activeFieldsChunks.length); - return hash64(root, activeFieldsRoot); + activeFieldsSingleChunk.fill(0); + activeFieldsSingleChunk.set(activeFields.uint8Array); + // 1 chunk for root, 1 chunk for activeFields + const chunkCount = 2; + merkleizeInto(mixInActiveFieldsChunkBytes, chunkCount, output, offset); + return; + } + + const chunkCount = Math.ceil(activeFields.uint8Array.length / 32); + merkleizeInto(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); + // 1 chunk for root, 1 chunk for activeFields + merkleizeInto(mixInActiveFieldsChunkBytes, 2, output, offset); } diff --git a/packages/ssz/src/type/uint.ts b/packages/ssz/src/type/uint.ts index 910310f4..81b47a18 100644 --- a/packages/ssz/src/type/uint.ts +++ b/packages/ssz/src/type/uint.ts @@ -133,6 +133,12 @@ export class UintNumberType extends BasicType { } } + value_toTree(value: number): Node { + const node = LeafNode.fromZero(); + node.setUint(this.byteLength, 0, value, this.clipInfinity); + return node; + } + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { const value = (node as LeafNode).getUint(this.byteLength, 0, this.clipInfinity); this.value_serializeToBytes(output, offset, value); diff --git a/packages/ssz/src/type/union.ts b/packages/ssz/src/type/union.ts index fbd7f97a..6a6117dd 100644 --- a/packages/ssz/src/type/union.ts +++ b/packages/ssz/src/type/union.ts @@ -4,16 +4,17 @@ import { Gindex, Node, Tree, - HashComputationLevel, + merkleizeInto, getHashComputations, + HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; -import {mixInLength} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ByteViews} from "./abstract"; import {CompositeType, isCompositeType} from "./composite"; import {addLengthNode, getLengthFromRootNode} from "./arrayBasic"; import {NoneType} from "./none"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -48,6 +49,12 @@ export class UnionType[]> extends CompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly maxSelector: number; @@ -85,6 +92,7 @@ export class UnionType[]> extends CompositeType< this.minSize = 1 + Math.min(...minLens); this.maxSize = 1 + Math.max(...maxLens); this.maxSelector = this.types.length - 1; + this.chunkBytesBuffer = new Uint8Array(32); } static named[]>(types: Types, opts: Require): UnionType { @@ -170,12 +178,21 @@ export class UnionType[]> extends CompositeType< // Merkleization hashTreeRoot(value: ValueOfTypes): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.selector); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: ValueOfTypes, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.selector, 32, 6); + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } - protected getRoots(value: ValueOfTypes): Uint8Array[] { - const valueRoot = this.types[value.selector].hashTreeRoot(value.value); - return [valueRoot]; + protected getChunkBytes(value: ValueOfTypes): Uint8Array { + this.types[value.selector].hashTreeRootInto(value.value, this.chunkBytesBuffer, 0); + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/vectorBasic.ts b/packages/ssz/src/type/vectorBasic.ts index d52a9405..bb189044 100644 --- a/packages/ssz/src/type/vectorBasic.ts +++ b/packages/ssz/src/type/vectorBasic.ts @@ -1,5 +1,5 @@ -import {Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; -import {maxChunksToDepth, splitIntoRootChunks} from "../util/merkleize"; +import {HashComputationLevel, Node, Tree} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ValueOf, ByteViews} from "./abstract"; @@ -59,6 +59,10 @@ export class VectorBasicType> this.minSize = this.fixedSize; this.maxSize = this.fixedSize; this.defaultLen = length; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = new Uint8Array( + this.maxChunkCount % 2 === 1 ? this.maxChunkCount * 32 + 32 : this.maxChunkCount * 32 + ); } static named>( @@ -146,11 +150,13 @@ export class VectorBasicType> // Merkleization - protected getRoots(value: ValueOf[]): Uint8Array[] { - const uint8Array = new Uint8Array(this.fixedSize); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const uint8Array = this.chunkBytesBuffer.subarray(0, this.fixedSize); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, this.length, {uint8Array, dataView}, 0, value); - return splitIntoRootChunks(uint8Array); + + // remaining bytes from this.fixedSize to this.chunkBytesBuffer.length must be zeroed + return this.chunkBytesBuffer; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/vectorComposite.ts b/packages/ssz/src/type/vectorComposite.ts index e1af8dd4..28990c43 100644 --- a/packages/ssz/src/type/vectorComposite.ts +++ b/packages/ssz/src/type/vectorComposite.ts @@ -11,9 +11,9 @@ import { tree_serializedSizeArrayComposite, tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, - value_getRootsArrayComposite, maxSizeArrayComposite, minSizeArrayComposite, + value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType, ArrayCompositeTreeView} from "../view/arrayComposite"; import {ArrayCompositeTreeViewDU} from "../viewDU/arrayComposite"; @@ -65,6 +65,10 @@ export class VectorCompositeType< this.minSize = minSizeArrayComposite(elementType, length); this.maxSize = maxSizeArrayComposite(elementType, length); this.defaultLen = length; + this.chunkBytesBuffer = + this.maxChunkCount % 2 === 1 + ? new Uint8Array(this.maxChunkCount * 32 + 32) + : new Uint8Array(this.maxChunkCount * 32); } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -153,8 +157,8 @@ export class VectorCompositeType< // Merkleization - protected getRoots(value: ValueOf[]): Uint8Array[] { - return value_getRootsArrayComposite(this.elementType, this.length, value); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + return value_getChunkBytesArrayComposite(this.elementType, this.length, value, this.chunkBytesBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/util/merkleize.ts b/packages/ssz/src/util/merkleize.ts index 073dea5d..932e80d7 100644 --- a/packages/ssz/src/util/merkleize.ts +++ b/packages/ssz/src/util/merkleize.ts @@ -1,5 +1,4 @@ -import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/index"; -import {zeroHash} from "@chainsafe/persistent-merkle-tree"; +import {hasher, zeroHash} from "@chainsafe/persistent-merkle-tree"; /** Dedicated property to cache hashTreeRoot of immutable CompositeType values */ export const symbolCachedPermanentRoot = Symbol("ssz_cached_permanent_root"); @@ -9,6 +8,28 @@ export type ValueWithCachedPermanentRoot = { [symbolCachedPermanentRoot]?: Uint8Array; }; +/** + * Cache a root for a ValueWithCachedPermanentRoot instance + * - if safeCache is true and output is 32 bytes and offset is 0, use output directly + * - if safeCache, use output subarray + * - otherwise, need to clone the root at output offset + */ +export function cacheRoot( + value: ValueWithCachedPermanentRoot, + output: Uint8Array, + offset: number, + safeCache: boolean +): void { + const cachedRoot = + safeCache && output.length === 32 && offset === 0 + ? output + : safeCache + ? output.subarray(offset, offset + 32) + : // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 + Uint8Array.prototype.slice.call(output, offset, offset + 32); + value[symbolCachedPermanentRoot] = cachedRoot; +} + export function hash64(bytes32A: Uint8Array, bytes32B: Uint8Array): Uint8Array { return hasher.digest64(bytes32A, bytes32B); } @@ -43,22 +64,6 @@ export function merkleize(chunks: Uint8Array[], padFor: number): Uint8Array { return chunks[0]; } -/** - * Split a long Uint8Array into Uint8Array of exactly 32 bytes - */ -export function splitIntoRootChunks(longChunk: Uint8Array): Uint8Array[] { - const chunkCount = Math.ceil(longChunk.length / 32); - const chunks = new Array(chunkCount); - - for (let i = 0; i < chunkCount; i++) { - const chunk = new Uint8Array(32); - chunk.set(longChunk.slice(i * 32, (i + 1) * 32)); - chunks[i] = chunk; - } - - return chunks; -} - /** @ignore */ export function mixInLength(root: Uint8Array, length: number): Uint8Array { const lengthBuf = Buffer.alloc(32); diff --git a/packages/ssz/test/perf/eth2/beaconBlock.test.ts b/packages/ssz/test/perf/eth2/beaconBlock.test.ts new file mode 100644 index 00000000..523e94ea --- /dev/null +++ b/packages/ssz/test/perf/eth2/beaconBlock.test.ts @@ -0,0 +1,94 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {ValueWithCachedPermanentRoot, symbolCachedPermanentRoot} from "../../../src/util/merkleize"; +import {deneb, ssz} from "../../lodestarTypes"; +import {preset} from "../../lodestarTypes/params"; +import {BitArray, toHexString} from "../../../src"; +const {MAX_ATTESTATIONS, MAX_DEPOSITS, MAX_VOLUNTARY_EXITS, MAX_BLS_TO_EXECUTION_CHANGES} = preset; + +describe("Benchmark BeaconBlock.hashTreeRoot()", function () { + setBenchOpts({ + minMs: 10_000, + }); + + const block = ssz.deneb.BeaconBlock.defaultValue(); + for (let i = 0; i < MAX_ATTESTATIONS; i++) { + block.body.attestations.push({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: 64}, () => true)), + data: { + slot: 1, + index: 1, + beaconBlockRoot: Buffer.alloc(32, 1), + source: { + epoch: 1, + root: Buffer.alloc(32, 1), + }, + target: { + epoch: 1, + root: Buffer.alloc(32, 1), + }, + }, + signature: Buffer.alloc(96, 1), + }); + } + for (let i = 0; i < MAX_DEPOSITS; i++) { + block.body.deposits.push({ + proof: ssz.phase0.Deposit.fields.proof.defaultValue(), + data: { + pubkey: Buffer.alloc(48, 1), + withdrawalCredentials: Buffer.alloc(32, 1), + amount: 32 * 1e9, + signature: Buffer.alloc(96, 1), + }, + }); + } + for (let i = 0; i < MAX_VOLUNTARY_EXITS; i++) { + block.body.voluntaryExits.push({ + signature: Buffer.alloc(96, 1), + message: { + epoch: 1, + validatorIndex: 1, + }, + }); + } + // common data on mainnet as of Jun 2024 + const numTransaction = 200; + const transactionLen = 500; + for (let i = 0; i < numTransaction; i++) { + block.body.executionPayload.transactions.push(Buffer.alloc(transactionLen, 1)); + } + for (let i = 0; i < MAX_BLS_TO_EXECUTION_CHANGES; i++) { + block.body.blsToExecutionChanges.push({ + signature: Buffer.alloc(96, 1), + message: { + validatorIndex: 1, + fromBlsPubkey: Buffer.alloc(48, 1), + toExecutionAddress: Buffer.alloc(20, 1), + }, + }); + } + + const root = ssz.deneb.BeaconBlock.hashTreeRoot(block); + console.log("BeaconBlock.hashTreeRoot() root", toHexString(root)); + itBench({ + id: `Deneb BeaconBlock.hashTreeRoot(), numTransaction=${numTransaction}`, + beforeEach: () => { + clearCachedRoots(block); + return block; + }, + fn: (block: deneb.BeaconBlock) => { + ssz.deneb.BeaconBlock.hashTreeRoot(block); + }, + }); +}); + +function clearCachedRoots(block: deneb.BeaconBlock): void { + (block as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + (block.body as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + const attestations = block.body.attestations; + for (const attestation of attestations) { + (attestation.data as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + } + for (const exit of block.body.voluntaryExits) { + (exit as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + } +} diff --git a/packages/ssz/test/perf/merkleize.test.ts b/packages/ssz/test/perf/merkleize.test.ts index b83a1f5d..a900015a 100644 --- a/packages/ssz/test/perf/merkleize.test.ts +++ b/packages/ssz/test/perf/merkleize.test.ts @@ -1,5 +1,6 @@ import {itBench} from "@dapplion/benchmark"; -import {bitLength} from "../../src/util/merkleize"; +import {bitLength, merkleize} from "../../src/util/merkleize"; +import {merkleizeInto} from "@chainsafe/persistent-merkle-tree"; describe("merkleize / bitLength", () => { for (const n of [50, 8000, 250000]) { @@ -13,6 +14,23 @@ describe("merkleize / bitLength", () => { } }); +describe("merkleize vs persistent-merkle-tree merkleizeInto", () => { + const chunkCounts = [4, 8, 16, 32]; + + for (const chunkCount of chunkCounts) { + const rootArr = Array.from({length: chunkCount}, (_, i) => Buffer.alloc(32, i)); + const roots = Buffer.concat(rootArr); + const result = Buffer.alloc(32); + itBench(`merkleizeInto ${chunkCount} chunks`, () => { + merkleizeInto(roots, chunkCount, result, 0); + }); + + itBench(`merkleize ${chunkCount} chunks`, () => { + merkleize(rootArr, chunkCount); + }); + } +}); + // Previous implementation, replaced by bitLength function bitLengthStr(n: number): number { const bitstring = n.toString(2); diff --git a/packages/ssz/test/spec/runValidTest.ts b/packages/ssz/test/spec/runValidTest.ts index 1bac7760..5ea219eb 100644 --- a/packages/ssz/test/spec/runValidTest.ts +++ b/packages/ssz/test/spec/runValidTest.ts @@ -101,13 +101,10 @@ export function runValidSszTest(type: Type, testData: ValidTestCaseData // 0x0000000000000000000000000000000000000000000000000000000000000000 if (process.env.RENDER_ROOTS) { if (type.isBasic) { - console.log("ROOTS Basic", toHexString(type.serialize(testDataValue))); + console.log("Chunk Bytes Basic", toHexString(type.serialize(testDataValue))); } else { - const roots = (type as CompositeType)["getRoots"](testDataValue); - console.log( - "ROOTS Composite", - roots.map((root) => toHexString(root)) - ); + const chunkBytes = (type as CompositeType)["getChunkBytes"](testDataValue); + console.log("Chunk Bytes Composite", toHexString(chunkBytes)); } } diff --git a/packages/ssz/test/unit/merkleize.test.ts b/packages/ssz/test/unit/merkleize.test.ts index 6b996c8c..d1e611b7 100644 --- a/packages/ssz/test/unit/merkleize.test.ts +++ b/packages/ssz/test/unit/merkleize.test.ts @@ -1,5 +1,6 @@ import {expect} from "chai"; -import {bitLength, maxChunksToDepth, nextPowerOf2} from "../../src/util/merkleize"; +import {bitLength, maxChunksToDepth, merkleize, mixInLength, nextPowerOf2} from "../../src/util/merkleize"; +import {merkleizeInto, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree"; describe("util / merkleize / bitLength", () => { const bitLengthByIndex = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4]; @@ -30,3 +31,35 @@ describe("util / merkleize / nextPowerOf2", () => { }); } }); + +describe("util / merkleize / mixInLength", () => { + const root = Buffer.alloc(32, 1); + const lengths = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + for (const length of lengths) { + it(`mixInLength(${length})`, () => { + const mixInLengthBuffer = Buffer.alloc(64); + mixInLengthBuffer.set(root, 0); + mixInLengthBuffer.writeUIntLE(length, 32, 6); + const finalRoot = new Uint8Array(32); + merkleizeInto(mixInLengthBuffer, 2, finalRoot, 0); + const expectedRoot = mixInLength(root, length); + expect(finalRoot).to.be.deep.equal(expectedRoot); + }); + } +}); + +describe("merkleize should be equal to merkleizeInto of hasher", () => { + const numNodes = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + for (const numNode of numNodes) { + it(`merkleize for ${numNode} nodes`, () => { + const nodes = Array.from({length: numNode}, (_, i) => LeafNode.fromRoot(Buffer.alloc(32, i))); + const data = Buffer.concat(nodes.map((node) => node.root)); + const padData = numNode % 2 === 1 ? Buffer.concat([data, zeroHash(0)]) : data; + const roots = nodes.map((node) => node.root); + const expectedRoot = Buffer.alloc(32); + const chunkCount = Math.max(numNode, 1); + merkleizeInto(padData, chunkCount, expectedRoot, 0); + expect(merkleize(roots, chunkCount)).to.be.deep.equal(expectedRoot); + }); + } +}); From 7ed3ced134c60cacd385c93d1504da5b96210300 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 6 Nov 2024 16:14:38 +0800 Subject: [PATCH 2/3] feat: consume merkleizeBlockArray --- packages/ssz/src/type/arrayComposite.ts | 20 +++--- packages/ssz/src/type/bitArray.ts | 14 ++-- packages/ssz/src/type/bitList.ts | 14 ++-- packages/ssz/src/type/byteArray.ts | 28 ++++---- packages/ssz/src/type/byteList.ts | 48 ++++++++++++-- packages/ssz/src/type/composite.ts | 12 ++-- packages/ssz/src/type/container.ts | 9 ++- packages/ssz/src/type/listBasic.ts | 32 +++++----- packages/ssz/src/type/listComposite.ts | 59 ++++++++++++----- packages/ssz/src/type/optional.ts | 25 ++++---- packages/ssz/src/type/profile.ts | 19 +++--- packages/ssz/src/type/stableContainer.ts | 33 +++++----- packages/ssz/src/type/union.ts | 23 +++---- packages/ssz/src/type/vectorBasic.ts | 13 ++-- packages/ssz/src/type/vectorComposite.ts | 11 ++-- packages/ssz/test/perf/merkleize.test.ts | 27 ++++++-- .../test/unit/byType/byteList/value.test.ts | 28 ++++++++ .../test/unit/byType/container/tree.test.ts | 64 ++----------------- .../unit/byType/listComposite/tree.test.ts | 17 ++++- 19 files changed, 277 insertions(+), 219 deletions(-) create mode 100644 packages/ssz/test/unit/byType/byteList/value.test.ts diff --git a/packages/ssz/src/type/arrayComposite.ts b/packages/ssz/src/type/arrayComposite.ts index 986b0e0a..d77e89dc 100644 --- a/packages/ssz/src/type/arrayComposite.ts +++ b/packages/ssz/src/type/arrayComposite.ts @@ -211,29 +211,29 @@ export function tree_deserializeFromBytesArrayComposite>( +export function value_getBlocksBytesArrayComposite>( elementType: ElementType, length: number, value: ValueOf[], - chunkBytesBuffer: Uint8Array + blocksBuffer: Uint8Array ): Uint8Array { - const isOddChunk = length % 2 === 1; - const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32; - if (chunkBytesLen > chunkBytesBuffer.length) { - throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`); + const blockBytesLen = Math.ceil(length / 2) * 64; + if (blockBytesLen > blocksBuffer.length) { + throw new Error(`blocksBuffer is too small: ${blocksBuffer.length} < ${blockBytesLen}`); } - const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen); + const blocksBytes = blocksBuffer.subarray(0, blockBytesLen); for (let i = 0; i < length; i++) { - elementType.hashTreeRootInto(value[i], chunkBytes, i * 32); + elementType.hashTreeRootInto(value[i], blocksBytes, i * 32); } + const isOddChunk = length % 2 === 1; if (isOddChunk) { // similar to append zeroHash(0) - chunkBytes.subarray(length * 32, chunkBytesLen).fill(0); + blocksBytes.subarray(length * 32, blockBytesLen).fill(0); } - return chunkBytes; + return blocksBytes; } function readOffsetsArrayComposite( diff --git a/packages/ssz/src/type/bitArray.ts b/packages/ssz/src/type/bitArray.ts index d485de27..469cd131 100644 --- a/packages/ssz/src/type/bitArray.ts +++ b/packages/ssz/src/type/bitArray.ts @@ -4,7 +4,7 @@ import {CompositeType, LENGTH_GINDEX} from "./composite"; import {BitArray} from "../value/bitArray"; import {BitArrayTreeView} from "../view/bitArray"; import {BitArrayTreeViewDU} from "../viewDU/bitArray"; -import {getChunkBytes} from "./byteArray"; +import {getBlocksBytes} from "./byteArray"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -40,15 +40,13 @@ export abstract class BitArrayType extends CompositeType this.chunkBytesBuffer.length) { + protected getBlocksBytes(value: BitArray): Uint8Array { + // reallocate this.blocksBuffer if needed + if (value.uint8Array.length > this.blocksBuffer.length) { const chunkCount = Math.ceil(value.bitLen / 8 / 32); - const chunkBytes = chunkCount * 32; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64); } - return getChunkBytes(value.uint8Array, this.chunkBytesBuffer); + return getBlocksBytes(value.uint8Array, this.blocksBuffer); } // Proofs diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index ba1c419a..c343ac48 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -1,7 +1,7 @@ import {allocUnsafe} from "@chainsafe/as-sha256"; import { getNodesAtDepth, - merkleizeInto, + merkleizeBlocksBytes, Node, packedNodeRootsToBytes, packedRootsBytesToNode, @@ -36,11 +36,11 @@ export class BitListType extends BitArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly limitBits: number, opts?: BitListOptions) { @@ -120,12 +120,12 @@ export class BitListType extends BitArrayType { } hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); // mixInLength this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/byteArray.ts b/packages/ssz/src/type/byteArray.ts index 78e6ae30..fc67037b 100644 --- a/packages/ssz/src/type/byteArray.ts +++ b/packages/ssz/src/type/byteArray.ts @@ -89,15 +89,13 @@ export abstract class ByteArrayType extends CompositeType this.chunkBytesBuffer.length) { + protected getBlocksBytes(value: ByteArray): Uint8Array { + // reallocate this.blocksBuffer if needed + if (value.length > this.blocksBuffer.length) { const chunkCount = Math.ceil(value.length / 32); - const chunkBytes = chunkCount * 32; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64); } - return getChunkBytes(value, this.chunkBytesBuffer); + return getBlocksBytes(value, this.blocksBuffer); } // Proofs @@ -162,15 +160,15 @@ export abstract class ByteArrayType extends CompositeType merkleBytesBuffer.length) { - throw new Error(`data length ${data.length} exceeds merkleBytesBuffer length ${merkleBytesBuffer.length}`); +export function getBlocksBytes(value: Uint8Array, blocksBuffer: Uint8Array): Uint8Array { + if (value.length > blocksBuffer.length) { + throw new Error(`data length ${value.length} exceeds blocksBuffer length ${blocksBuffer.length}`); } - merkleBytesBuffer.set(data); - const valueLen = data.length; - const chunkByteLen = Math.ceil(valueLen / 64) * 64; + blocksBuffer.set(value); + const valueLen = value.length; + const blockByteLen = Math.ceil(valueLen / 64) * 64; // all padding bytes must be zero, this is similar to set zeroHash(0) - merkleBytesBuffer.subarray(valueLen, chunkByteLen).fill(0); - return merkleBytesBuffer.subarray(0, chunkByteLen); + blocksBuffer.subarray(valueLen, blockByteLen).fill(0); + return blocksBuffer.subarray(0, blockByteLen); } diff --git a/packages/ssz/src/type/byteList.ts b/packages/ssz/src/type/byteList.ts index 53c46d39..0ceaab7a 100644 --- a/packages/ssz/src/type/byteList.ts +++ b/packages/ssz/src/type/byteList.ts @@ -4,7 +4,8 @@ import { Node, packedNodeRootsToBytes, packedRootsBytesToNode, - merkleizeInto, + merkleizeBlocksBytes, + merkleizeBlockArray, } from "@chainsafe/persistent-merkle-tree"; import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; @@ -40,11 +41,13 @@ export class ByteListType extends ByteArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly blockArray: Uint8Array[] = []; + private blockBytesLen = 0; + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly limitBytes: number, opts?: ByteListOptions) { @@ -106,13 +109,44 @@ export class ByteListType extends ByteArrayType { return root; } + /** + * Use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + */ hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // should not call super.hashTreeRoot() here + // use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + // reallocate this.blockArray if needed + if (value.length > this.blockBytesLen) { + const newBlockCount = Math.ceil(value.length / 64); + // this.blockBytesLen should be a multiple of 64 + const oldBlockCount = Math.ceil(this.blockBytesLen / 64); + const blockDiff = newBlockCount - oldBlockCount; + const newBlocksBytes = new Uint8Array(blockDiff * 64); + for (let i = 0; i < blockDiff; i++) { + this.blockArray.push(newBlocksBytes.subarray(i * 64, (i + 1) * 64)); + this.blockBytesLen += 64; + } + } + + // populate this.blockArray + for (let i = 0; i < value.length; i += 64) { + const block = this.blockArray[i / 64]; + // zero out the last block if it's over value.length + if (i + 64 > value.length) { + block.fill(0); + } + block.set(value.subarray(i, Math.min(i + 64, value.length))); + } + + // compute hashTreeRoot + const blockLimit = Math.ceil(value.length / 64); + merkleizeBlockArray(this.blockArray, blockLimit, this.maxChunkCount, this.mixInLengthBlockBytes, 0); + // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/composite.ts b/packages/ssz/src/type/composite.ts index 9f33ca5e..ce70be4b 100644 --- a/packages/ssz/src/type/composite.ts +++ b/packages/ssz/src/type/composite.ts @@ -8,7 +8,7 @@ import { Proof, ProofType, Tree, - merkleizeInto, + merkleizeBlocksBytes, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; import {byteArrayEquals} from "../util/byteArray"; @@ -61,7 +61,7 @@ export abstract class CompositeType extends Type { * Required for ContainerNodeStruct to ensure no dangerous types are constructed. */ abstract readonly isViewMutable: boolean; - protected chunkBytesBuffer = new Uint8Array(0); + protected blocksBuffer = new Uint8Array(0); constructor( /** @@ -238,8 +238,8 @@ export abstract class CompositeType extends Type { } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, output, offset); + const blocksBuffer = this.getBlocksBytes(value); + merkleizeBlocksBytes(blocksBuffer, this.maxChunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } @@ -258,10 +258,10 @@ export abstract class CompositeType extends Type { // to hashObject and back. /** - * Get merkle bytes of each value, the returned Uint8Array should be multiple of 64 bytes. + * Get multiple SHA256 blocks, each is 64 bytes long. * If chunk count is not even, need to append zeroHash(0) */ - protected abstract getChunkBytes(value: V): Uint8Array; + protected abstract getBlocksBytes(value: V): Uint8Array; // Proofs API diff --git a/packages/ssz/src/type/container.ts b/packages/ssz/src/type/container.ts index 1ed46a89..2d6505ea 100644 --- a/packages/ssz/src/type/container.ts +++ b/packages/ssz/src/type/container.ts @@ -131,8 +131,7 @@ export class ContainerType>> extends this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); const fieldBytes = this.fieldsEntries.length * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -275,13 +274,13 @@ export class ContainerType>> extends // Merkleization - protected getChunkBytes(struct: ValueOfFields): Uint8Array { + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType} = this.fieldsEntries[i]; - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, i * 32); } // remaining bytes are zeroed as we never write them - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/listBasic.ts b/packages/ssz/src/type/listBasic.ts index 4dd63f08..9af933d7 100644 --- a/packages/ssz/src/type/listBasic.ts +++ b/packages/ssz/src/type/listBasic.ts @@ -1,4 +1,4 @@ -import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {HashComputationLevel, LeafNode, Node, Tree, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree"; import {ValueOf} from "./abstract"; import {BasicType} from "./basic"; import {ByteViews} from "./composite"; @@ -47,11 +47,11 @@ export class ListBasicType> readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly defaultLen = 0; @@ -193,34 +193,34 @@ export class ListBasicType> } } - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } } - protected getChunkBytes(value: ValueOf[]): Uint8Array { + protected getBlocksBytes(value: ValueOf[]): Uint8Array { const byteLen = this.value_serializedSize(value); - const chunkByteLen = Math.ceil(byteLen / 64) * 64; - // reallocate this.verkleBytes if needed - if (byteLen > this.chunkBytesBuffer.length) { + const blockByteLen = Math.ceil(byteLen / 64) * 64; + // reallocate this.blocksBuffer if needed + if (byteLen > this.blocksBuffer.length) { // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = new Uint8Array(chunkByteLen); + this.blocksBuffer = new Uint8Array(blockByteLen); } - const chunkBytes = this.chunkBytesBuffer.subarray(0, chunkByteLen); - const uint8Array = chunkBytes.subarray(0, byteLen); + const blockBytes = this.blocksBuffer.subarray(0, blockByteLen); + const uint8Array = blockBytes.subarray(0, byteLen); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value); // all padding bytes must be zero, this is similar to set zeroHash(0) - this.chunkBytesBuffer.subarray(byteLen, chunkByteLen).fill(0); - return chunkBytes; + this.blocksBuffer.subarray(byteLen, blockByteLen).fill(0); + return blockBytes; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/listComposite.ts b/packages/ssz/src/type/listComposite.ts index 821b9504..5487f700 100644 --- a/packages/ssz/src/type/listComposite.ts +++ b/packages/ssz/src/type/listComposite.ts @@ -1,4 +1,10 @@ -import {HashComputationLevel, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import { + HashComputationLevel, + Node, + Tree, + merkleizeBlocksBytes, + merkleizeBlockArray, +} from "@chainsafe/persistent-merkle-tree"; import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; @@ -13,7 +19,6 @@ import { tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, maxSizeArrayComposite, - value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType} from "../view/arrayComposite"; import {ListCompositeTreeView} from "../view/listComposite"; @@ -52,11 +57,12 @@ export class ListCompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly blockArray: Uint8Array[] = []; + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly defaultLen = 0; @@ -200,25 +206,48 @@ export class ListCompositeType< } } - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // should not call super.hashTreeRootInto() here + // use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + // reallocate this.blockArray if needed + if (value.length > this.blockArray.length) { + const blockDiff = value.length - this.blockArray.length; + const newBlocksBytes = new Uint8Array(blockDiff * 64); + for (let i = 0; i < blockDiff; i++) { + this.blockArray.push(newBlocksBytes.subarray(i * 64, (i + 1) * 64)); + } + } + + // populate this.blockArray + for (let i = 0; i < value.length; i++) { + // 2 values share a block + const block = this.blockArray[Math.floor(i / 2)]; + const offset = i % 2 === 0 ? 0 : 32; + this.elementType.hashTreeRootInto(value[i], block, offset); + } + + const blockLimit = Math.ceil(value.length / 2); + // zero out the last block if needed + if (value.length % 2 === 1) { + this.blockArray[blockLimit - 1].fill(0, 32); + } + + // compute hashTreeRoot + merkleizeBlockArray(this.blockArray, blockLimit, this.maxChunkCount, this.mixInLengthBlockBytes, 0); + // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } } - protected getChunkBytes(value: ValueOf[]): Uint8Array { - const byteLen = value.length * 32; - const chunkByteLen = this.chunkBytesBuffer.byteLength; - if (byteLen > chunkByteLen) { - this.chunkBytesBuffer = new Uint8Array(Math.ceil(byteLen / 64) * 64); - } - return value_getChunkBytesArrayComposite(this.elementType, value.length, value, this.chunkBytesBuffer); + protected getBlocksBytes(): Uint8Array { + // we use merkleizeBlockArray for hashTreeRoot() computation + throw Error("getBlockBytes should not be called for ListCompositeType"); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 3d2925fb..1473e6d7 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -1,7 +1,7 @@ import { concatGindices, Gindex, - merkleizeInto, + merkleizeBlocksBytes, Node, Tree, zeroNode, @@ -48,11 +48,11 @@ export class OptionalType> extends CompositeTy readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly elementType: ElementType, opts?: OptionalOpts) { @@ -66,7 +66,8 @@ export class OptionalType> extends CompositeTy this.minSize = 0; // Max size includes prepended 0x01 byte this.maxSize = elementType.maxSize + 1; - this.chunkBytesBuffer = new Uint8Array(32); + // maxChunkCount = 1 so this.blocksBuffer.length = 32 in this case + this.blocksBuffer = new Uint8Array(32); } static named>( @@ -185,21 +186,21 @@ export class OptionalType> extends CompositeTy } hashTreeRootInto(value: ValueOfType, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); const selector = value === null ? 0 : 1; this.mixInLengthBuffer.writeUIntLE(selector, 32, 6); // one for hashTreeRoot(value), one for selector const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } - protected getChunkBytes(value: ValueOfType): Uint8Array { + protected getBlocksBytes(value: ValueOfType): Uint8Array { if (value === null) { - this.chunkBytesBuffer.fill(0); + this.blocksBuffer.fill(0); } else { - this.elementType.hashTreeRootInto(value, this.chunkBytesBuffer, 0); + this.elementType.hashTreeRootInto(value, this.blocksBuffer, 0); } - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts index 1ac440d7..06b2cf76 100644 --- a/packages/ssz/src/type/profile.ts +++ b/packages/ssz/src/type/profile.ts @@ -6,7 +6,7 @@ import { Gindex, toGindex, concatGindices, - merkleizeInto, + merkleizeBlocksBytes, getNode, BranchNode, zeroHash, @@ -158,8 +158,7 @@ export class ProfileType>> extends C this.TreeView = opts?.getProfileTreeViewClass?.(this) ?? getProfileTreeViewClass(this); this.TreeViewDU = opts?.getProfileTreeViewDUClass?.(this) ?? getProfileTreeViewDUClass(this); const fieldBytes = this.activeFields.bitLen * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -378,8 +377,8 @@ export class ProfileType>> extends C } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + const blocksBytes = this.getBlocksBytes(value); + merkleizeBlocksBytes(blocksBytes, this.maxChunkCount, this.tempRoot, 0); mixInActiveFields(this.tempRoot, this.activeFields, output, offset); if (this.cachePermanentRootStruct) { @@ -387,18 +386,18 @@ export class ProfileType>> extends C } } - protected getChunkBytes(struct: ValueOfFields): Uint8Array { - this.chunkBytesBuffer.fill(0); + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { + this.blocksBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, chunkIndex, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - this.chunkBytesBuffer.set(zeroHash(0), chunkIndex * 32); + this.blocksBuffer.set(zeroHash(0), chunkIndex * 32); } else { - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, chunkIndex * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, chunkIndex * 32); } } // remaining bytes are zeroed as we never write them - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index 35415ad8..b64ed331 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -11,7 +11,7 @@ import { getNode, zeroNode, zeroHash, - merkleizeInto, + merkleizeBlocksBytes, countToDepth, getNodeH, setNode, @@ -150,8 +150,7 @@ export class StableContainerType>> e this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); const fieldBytes = this.fieldsEntries.length * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -351,8 +350,8 @@ export class StableContainerType>> e } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + const blockBytes = this.getBlocksBytes(value); + merkleizeBlocksBytes(blockBytes, this.maxChunkCount, this.tempRoot, 0); // compute active field bitvector const activeFields = BitArray.fromBoolArray([ ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), @@ -365,18 +364,18 @@ export class StableContainerType>> e } } - protected getChunkBytes(struct: ValueOfFields): Uint8Array { - this.chunkBytesBuffer.fill(0); + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { + this.blocksBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - this.chunkBytesBuffer.set(zeroHash(0), i * 32); + this.blocksBuffer.set(zeroHash(0), i * 32); } else { - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, i * 32); } } - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs @@ -817,24 +816,24 @@ export function setActiveField(rootNode: Node, bitLen: number, fieldIndex: numbe return new BranchNode(rootNode.left, newActiveFieldsNode); } -// This is a global buffer to avoid creating a new one for each call to getChunkBytes -const mixInActiveFieldsChunkBytes = new Uint8Array(64); -const activeFieldsSingleChunk = mixInActiveFieldsChunkBytes.subarray(32); +// This is a global buffer to avoid creating a new one for each call to getBlocksBytes +const mixInActiveFieldsBlockBytes = new Uint8Array(64); +const activeFieldsSingleChunk = mixInActiveFieldsBlockBytes.subarray(32); export function mixInActiveFields(root: Uint8Array, activeFields: BitArray, output: Uint8Array, offset: number): void { // fast path for depth 1, the bitvector fits in one chunk - mixInActiveFieldsChunkBytes.set(root, 0); + mixInActiveFieldsBlockBytes.set(root, 0); if (activeFields.bitLen <= 256) { activeFieldsSingleChunk.fill(0); activeFieldsSingleChunk.set(activeFields.uint8Array); // 1 chunk for root, 1 chunk for activeFields const chunkCount = 2; - merkleizeInto(mixInActiveFieldsChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(mixInActiveFieldsBlockBytes, chunkCount, output, offset); return; } const chunkCount = Math.ceil(activeFields.uint8Array.length / 32); - merkleizeInto(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); + merkleizeBlocksBytes(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); // 1 chunk for root, 1 chunk for activeFields - merkleizeInto(mixInActiveFieldsChunkBytes, 2, output, offset); + merkleizeBlocksBytes(mixInActiveFieldsBlockBytes, 2, output, offset); } diff --git a/packages/ssz/src/type/union.ts b/packages/ssz/src/type/union.ts index 6a6117dd..908d9604 100644 --- a/packages/ssz/src/type/union.ts +++ b/packages/ssz/src/type/union.ts @@ -4,7 +4,7 @@ import { Gindex, Node, Tree, - merkleizeInto, + merkleizeBlocksBytes, getHashComputations, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; @@ -49,11 +49,11 @@ export class UnionType[]> extends CompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly maxSelector: number; @@ -92,7 +92,8 @@ export class UnionType[]> extends CompositeType< this.minSize = 1 + Math.min(...minLens); this.maxSize = 1 + Math.max(...maxLens); this.maxSelector = this.types.length - 1; - this.chunkBytesBuffer = new Uint8Array(32); + // maxChunkCount = 1 so this.blocksBuffer.length = 32 in this case + this.blocksBuffer = new Uint8Array(32); } static named[]>(types: Types, opts: Require): UnionType { @@ -184,15 +185,15 @@ export class UnionType[]> extends CompositeType< } hashTreeRootInto(value: ValueOfTypes, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); this.mixInLengthBuffer.writeUIntLE(value.selector, 32, 6); const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } - protected getChunkBytes(value: ValueOfTypes): Uint8Array { - this.types[value.selector].hashTreeRootInto(value.value, this.chunkBytesBuffer, 0); - return this.chunkBytesBuffer; + protected getBlocksBytes(value: ValueOfTypes): Uint8Array { + this.types[value.selector].hashTreeRootInto(value.value, this.blocksBuffer, 0); + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/vectorBasic.ts b/packages/ssz/src/type/vectorBasic.ts index bb189044..0c528c96 100644 --- a/packages/ssz/src/type/vectorBasic.ts +++ b/packages/ssz/src/type/vectorBasic.ts @@ -59,10 +59,7 @@ export class VectorBasicType> this.minSize = this.fixedSize; this.maxSize = this.fixedSize; this.defaultLen = length; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = new Uint8Array( - this.maxChunkCount % 2 === 1 ? this.maxChunkCount * 32 + 32 : this.maxChunkCount * 32 - ); + this.blocksBuffer = new Uint8Array(Math.ceil(this.maxChunkCount / 2) * 64); } static named>( @@ -150,13 +147,13 @@ export class VectorBasicType> // Merkleization - protected getChunkBytes(value: ValueOf[]): Uint8Array { - const uint8Array = this.chunkBytesBuffer.subarray(0, this.fixedSize); + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + const uint8Array = this.blocksBuffer.subarray(0, this.fixedSize); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, this.length, {uint8Array, dataView}, 0, value); - // remaining bytes from this.fixedSize to this.chunkBytesBuffer.length must be zeroed - return this.chunkBytesBuffer; + // remaining bytes from this.fixedSize to this.blocksBuffer.length must be zeroed + return this.blocksBuffer; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/vectorComposite.ts b/packages/ssz/src/type/vectorComposite.ts index 28990c43..908f4f9b 100644 --- a/packages/ssz/src/type/vectorComposite.ts +++ b/packages/ssz/src/type/vectorComposite.ts @@ -13,7 +13,7 @@ import { tree_serializeToBytesArrayComposite, maxSizeArrayComposite, minSizeArrayComposite, - value_getChunkBytesArrayComposite, + value_getBlocksBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType, ArrayCompositeTreeView} from "../view/arrayComposite"; import {ArrayCompositeTreeViewDU} from "../viewDU/arrayComposite"; @@ -65,10 +65,7 @@ export class VectorCompositeType< this.minSize = minSizeArrayComposite(elementType, length); this.maxSize = maxSizeArrayComposite(elementType, length); this.defaultLen = length; - this.chunkBytesBuffer = - this.maxChunkCount % 2 === 1 - ? new Uint8Array(this.maxChunkCount * 32 + 32) - : new Uint8Array(this.maxChunkCount * 32); + this.blocksBuffer = new Uint8Array(Math.ceil(this.maxChunkCount / 2) * 64); } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -157,8 +154,8 @@ export class VectorCompositeType< // Merkleization - protected getChunkBytes(value: ValueOf[]): Uint8Array { - return value_getChunkBytesArrayComposite(this.elementType, this.length, value, this.chunkBytesBuffer); + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + return value_getBlocksBytesArrayComposite(this.elementType, this.length, value, this.blocksBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/test/perf/merkleize.test.ts b/packages/ssz/test/perf/merkleize.test.ts index a900015a..af70868e 100644 --- a/packages/ssz/test/perf/merkleize.test.ts +++ b/packages/ssz/test/perf/merkleize.test.ts @@ -1,6 +1,6 @@ import {itBench} from "@dapplion/benchmark"; import {bitLength, merkleize} from "../../src/util/merkleize"; -import {merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {merkleizeBlockArray, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree"; describe("merkleize / bitLength", () => { for (const n of [50, 8000, 250000]) { @@ -14,20 +14,33 @@ describe("merkleize / bitLength", () => { } }); -describe("merkleize vs persistent-merkle-tree merkleizeInto", () => { - const chunkCounts = [4, 8, 16, 32]; +describe("merkleize vs persistent-merkle-tree merkleizeBlocksBytes", () => { + const chunkCounts = [32, 128, 512, 1024]; for (const chunkCount of chunkCounts) { const rootArr = Array.from({length: chunkCount}, (_, i) => Buffer.alloc(32, i)); - const roots = Buffer.concat(rootArr); + const blocksBytes = Buffer.concat(rootArr); + if (blocksBytes.length % 64 !== 0) { + throw new Error("blockBytes length must be a multiple of 64"); + } + const blockArray: Uint8Array[] = []; + for (let i = 0; i < blocksBytes.length; i += 64) { + blockArray.push(blocksBytes.slice(i, i + 64)); + } + const result = Buffer.alloc(32); - itBench(`merkleizeInto ${chunkCount} chunks`, () => { - merkleizeInto(roots, chunkCount, result, 0); - }); itBench(`merkleize ${chunkCount} chunks`, () => { merkleize(rootArr, chunkCount); }); + + itBench(`merkleizeBlocksBytes ${chunkCount} chunks`, () => { + merkleizeBlocksBytes(blocksBytes, chunkCount, result, 0); + }); + + itBench(`merkleizeBlockArray ${chunkCount} chunks`, () => { + merkleizeBlockArray(blockArray, blockArray.length, chunkCount, result, 0); + }); } }); diff --git a/packages/ssz/test/unit/byType/byteList/value.test.ts b/packages/ssz/test/unit/byType/byteList/value.test.ts new file mode 100644 index 00000000..0033443f --- /dev/null +++ b/packages/ssz/test/unit/byType/byteList/value.test.ts @@ -0,0 +1,28 @@ +import {expect} from "chai"; +import {ByteListType} from "../../../../src"; + +describe("ByteListValue", () => { + const type = new ByteListType(1024); + + it("should zero out the last sha256 block if it's over value.length", () => { + const value = Buffer.alloc(65, 1); + const expectedRoot = type.hashTreeRoot(value); + // now hash another value which make the cached blocks non zero + type.hashTreeRoot(Buffer.alloc(1024, 2)); + const actualRoot = type.hashTreeRoot(value); + expect(actualRoot).to.deep.equal(expectedRoot); + }); + + it("should increase blockArray size if needed", () => { + const value0 = Buffer.alloc(65, 1); + const expectedRoot0 = type.hashTreeRoot(value0); + const value1 = Buffer.alloc(1024, 3); + const expectedRoot1 = type.hashTreeRoot(value1); + // now increase block array size + type.hashTreeRoot(Buffer.alloc(1024, 2)); + + // hash again + expect(type.hashTreeRoot(value0)).to.deep.equal(expectedRoot0); + expect(type.hashTreeRoot(value1)).to.deep.equal(expectedRoot1); + }); +}); diff --git a/packages/ssz/test/unit/byType/container/tree.test.ts b/packages/ssz/test/unit/byType/container/tree.test.ts index 83fad27a..5c3bbd54 100644 --- a/packages/ssz/test/unit/byType/container/tree.test.ts +++ b/packages/ssz/test/unit/byType/container/tree.test.ts @@ -1,5 +1,4 @@ import {expect} from "chai"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; import { BitArray, BitListType, @@ -13,7 +12,6 @@ import { ListCompositeType, NoneType, toHexString, - Type, UintNumberType, UnionType, ValueOf, @@ -22,7 +20,6 @@ import { } from "../../../../src"; import {uint64NumInfType, uint64NumType} from "../../../utils/primitiveTypes"; import {runViewTestMutation} from "../runViewTestMutation"; -import {upgradeToNewType} from "../../../../src/util/upgrade"; // Test both ContainerType, ContainerNodeStructType only if // - All fields are immutable @@ -244,7 +241,7 @@ describe("ContainerViewDU batchHashTreeRoot", function () { a: uint64NumType, b: new BooleanType(), c: unionType, - d: new ByteListType(64), + d: new ByteListType(1024), e: new ByteVectorType(64), // a child container type f: childContainerType, @@ -262,7 +259,8 @@ describe("ContainerViewDU batchHashTreeRoot", function () { a: 10, b: true, c: {selector: 1, value: 100}, - d: Buffer.alloc(64, 2), + // make this not divisible by 64 to test edge case + d: Buffer.alloc(65, 2), e: Buffer.alloc(64, 1), f: {f0: 100, f1: 101}, g: {g0: 100, g1: 101}, @@ -274,6 +272,7 @@ describe("ContainerViewDU batchHashTreeRoot", function () { m: BitArray.fromSingleBit(4, 1), }; const expectedRoot = parentContainerType.toView(value).hashTreeRoot(); + expect(parentContainerType.hashTreeRoot(value)).to.be.deep.equal(expectedRoot); it("fresh ViewDU", () => { expect(parentContainerType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot); @@ -330,9 +329,10 @@ describe("ContainerViewDU batchHashTreeRoot", function () { it("full hash then modify ByteListType", () => { const viewDU = parentContainerType.toViewDU(value); + viewDU.d = Buffer.alloc(1024, 3); viewDU.batchHashTreeRoot(); - // this takes more than 1 chunk so the resulting node is a branch node - viewDU.d = viewDU.d.slice(); + // set back to the original value, this takes more than 1 chunk so the resulting node is a branch node + viewDU.d = Buffer.alloc(65, 2); expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); // assign again but commit before batchHashTreeRoot() @@ -638,53 +638,3 @@ describe("ContainerNodeStruct batchHashTreeRoot", function () { expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); }); }); - -describe("upgradeToNewType utility", () => { - const numFields = [2, 7, 15, 17, 31, 33, 63, 65, 127, 129]; - for (const [i, numField] of numFields.entries()) { - it(`upgradeToNewType with ${numField} fields`, () => { - const fields: Record> = {}; - for (let j = 0; j < numField; j++) { - fields[`f${j}`] = uint64NumInfType; - } - const oldType = new ContainerType(fields); - const view = oldType.defaultView(); - const viewDU = oldType.defaultViewDU(); - for (let j = 0; j < numField; j++) { - (view as Record)[`f${j}`] = j; - (viewDU as Record)[`f${j}`] = j; - } - - for (let j = i + 1; j < numFields.length; j++) { - const newFields: Record> = {}; - for (let k = 0; k < numFields[j]; k++) { - (newFields as Record>)[`f${k}`] = uint64NumInfType; - } - - const newType = new ContainerType(newFields); - const newView = newType.getView(new Tree(upgradeToNewType(view.node, oldType, newType))); - // commit view DU to make sure the view is updated before accessing viewDU.node - viewDU.commit(); - const newViewDU = newType.getViewDU(upgradeToNewType(viewDU.node, oldType, newType)); - for (let k = i + 1; k < numFields[j]; k++) { - (newView as Record)[`f${k}`] = k; - (newViewDU as Record)[`f${k}`] = k; - } - newViewDU.commit(); - - const expectedValue = newType.defaultValue(); - for (let k = 0; k < numFields[j]; k++) { - (expectedValue as Record)[`f${k}`] = k; - } - const expectedViewDU = newType.toViewDU(expectedValue); - - expect(newView.toValue()).to.be.deep.equal(expectedValue); - expect(newView.hashTreeRoot()).to.be.deep.equal(expectedViewDU.hashTreeRoot()); - expect(newView.serialize()).to.be.deep.equal(expectedViewDU.serialize()); - expect(newViewDU.toValue()).to.be.deep.equal(expectedValue); - expect(newViewDU.hashTreeRoot()).to.be.deep.equal(expectedViewDU.hashTreeRoot()); - expect(newViewDU.serialize()).to.be.deep.equal(expectedViewDU.serialize()); - } - }); - } -}); diff --git a/packages/ssz/test/unit/byType/listComposite/tree.test.ts b/packages/ssz/test/unit/byType/listComposite/tree.test.ts index 95b39746..f428365b 100644 --- a/packages/ssz/test/unit/byType/listComposite/tree.test.ts +++ b/packages/ssz/test/unit/byType/listComposite/tree.test.ts @@ -226,7 +226,21 @@ describe("ListCompositeType.sliceFrom", () => { } }); -describe("ListCompositeType batchHashTreeRoot", () => { +describe("ListCompositeType hashTreeRoot", () => { + it("shouldzero out the last sha256 block", () => { + const listType = new ListCompositeType(ssz.Root, 1024); + const value0 = Array.from({length: 65}, (_, i) => Buffer.alloc(32, i)); + const value1 = Array.from({length: 120}, (_, i) => Buffer.alloc(32, i)); + const expectedRoot0 = listType.hashTreeRoot(value0); + const expectedRoot1 = listType.hashTreeRoot(value1); + // now increase block array size + listType.hashTreeRoot(Array.from({length: 1024}, () => Buffer.alloc(32, 3))); + expect(listType.hashTreeRoot(value0)).to.deep.equal(expectedRoot0); + expect(listType.hashTreeRoot(value1)).to.deep.equal(expectedRoot1); + }); +}); + +describe("ListCompositeType ViewDU batchHashTreeRoot", () => { const value = [ {a: 1, b: 2}, {a: 3, b: 4}, @@ -242,6 +256,7 @@ describe("ListCompositeType batchHashTreeRoot", () => { for (const list of [listOfContainersType, listOfContainersType2]) { const typeName = list.typeName; const expectedRoot = list.toView(value).hashTreeRoot(); + expect(listOfContainersType2.hashTreeRoot(value)).to.be.deep.equal(expectedRoot); it(`${typeName} - fresh ViewDU`, () => { expect(listOfContainersType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot); From d3821eefeb8b734c79518c496ba6d08976003158 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sat, 9 Nov 2024 10:26:03 +0800 Subject: [PATCH 3/3] fix: lint in ssz package --- packages/ssz/test/spec/runValidTest.ts | 4 ++-- packages/ssz/test/unit/merkleize.test.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ssz/test/spec/runValidTest.ts b/packages/ssz/test/spec/runValidTest.ts index 5ea219eb..2385d191 100644 --- a/packages/ssz/test/spec/runValidTest.ts +++ b/packages/ssz/test/spec/runValidTest.ts @@ -103,8 +103,8 @@ export function runValidSszTest(type: Type, testData: ValidTestCaseData if (type.isBasic) { console.log("Chunk Bytes Basic", toHexString(type.serialize(testDataValue))); } else { - const chunkBytes = (type as CompositeType)["getChunkBytes"](testDataValue); - console.log("Chunk Bytes Composite", toHexString(chunkBytes)); + const blocksBytes = (type as CompositeType)["getBlocksBytes"](testDataValue); + console.log("Blocks Bytes Composite", toHexString(blocksBytes)); } } diff --git a/packages/ssz/test/unit/merkleize.test.ts b/packages/ssz/test/unit/merkleize.test.ts index d1e611b7..94be7d64 100644 --- a/packages/ssz/test/unit/merkleize.test.ts +++ b/packages/ssz/test/unit/merkleize.test.ts @@ -1,6 +1,6 @@ import {expect} from "chai"; import {bitLength, maxChunksToDepth, merkleize, mixInLength, nextPowerOf2} from "../../src/util/merkleize"; -import {merkleizeInto, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree"; +import {merkleizeBlocksBytes, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree"; describe("util / merkleize / bitLength", () => { const bitLengthByIndex = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4]; @@ -41,7 +41,7 @@ describe("util / merkleize / mixInLength", () => { mixInLengthBuffer.set(root, 0); mixInLengthBuffer.writeUIntLE(length, 32, 6); const finalRoot = new Uint8Array(32); - merkleizeInto(mixInLengthBuffer, 2, finalRoot, 0); + merkleizeBlocksBytes(mixInLengthBuffer, 2, finalRoot, 0); const expectedRoot = mixInLength(root, length); expect(finalRoot).to.be.deep.equal(expectedRoot); }); @@ -58,7 +58,7 @@ describe("merkleize should be equal to merkleizeInto of hasher", () => { const roots = nodes.map((node) => node.root); const expectedRoot = Buffer.alloc(32); const chunkCount = Math.max(numNode, 1); - merkleizeInto(padData, chunkCount, expectedRoot, 0); + merkleizeBlocksBytes(padData, chunkCount, expectedRoot, 0); expect(merkleize(roots, chunkCount)).to.be.deep.equal(expectedRoot); }); }