feat: batch hash tree root

packages/persistent-merkle-tree/src/packedNode.ts

@@ -18,9 +18,16 @@ export function packedRootsBytesToNode(depth: number, dataView: DataView, start:
  *
  *    h0     h1     h2     h3     h4     h5     h6     h7
  * |------|------|------|------|------|------|------|------|
+ *
+ * @param values list of uint64 numbers
+ * @param leafNodes optional list of LeafNodes to reuse
  */
-export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] {
-  const leafNodes = new Array<LeafNode>(Math.ceil(values.length / 4));
+export function packedUintNum64sToLeafNodes(values: number[], leafNodes?: LeafNode[]): LeafNode[] {
+  const nodeCount = Math.ceil(values.length / 4);
+  if (leafNodes && leafNodes.length !== nodeCount) {
+    throw new Error(`Invalid leafNode length: ${leafNodes.length} !== ${nodeCount}`);
+  }
+  leafNodes = leafNodes ?? new Array<LeafNode>(Math.ceil(values.length / 4));
   for (let i = 0; i < values.length; i++) {
     const nodeIndex = Math.floor(i / 4);
     const leafNode = leafNodes[nodeIndex] ?? new LeafNode(0, 0, 0, 0, 0, 0, 0, 0);

packages/persistent-merkle-tree/test/perf/hasher.test.ts

@@ -79,7 +79,7 @@ describe("hashtree", function () {
   });
 
   itBench({
-    id: `executeHashComputations`,
+    id: `executeHashComputations - hashtree`,
     beforeEach: () => {
       const [tree] = buildComparisonTrees(16);
       return tree;
@@ -92,7 +92,7 @@ describe("hashtree", function () {
   });
 
   itBench({
-    id: `get root`,
+    id: `root - hashtree`,
     beforeEach: () => {
       const [tree] = buildComparisonTrees(16);
       setHasher(hashtreeHasher);

packages/persistent-merkle-tree/test/perf/node.test.ts

@@ -1,6 +1,6 @@
 import {itBench} from "@dapplion/benchmark";
 import {BranchNode, getNodeH, LeafNode} from "../../src/node";
-import {countToDepth, getHashComputations, HashComputation, subtreeFillToContents} from "../../src";
+import {countToDepth, getHashComputations, subtreeFillToContents} from "../../src";
 import {batchHash} from "../utils/batchHash";
 
 describe("HashObject LeafNode", () => {

packages/simpleserialize.com/package.json

@@ -25,6 +25,7 @@
     "eyzy-tree": "^0.2.2",
     "file-saver": "^2.0.5",
     "js-yaml": "^4.1.0",
+    "null-loader": "^4.0.1",
     "react": "^17.0.2",
     "react-alert": "^7.0.1",
     "react-alert-template-basic": "^1.0.0",

packages/simpleserialize.com/webpack.config.js

@@ -1,22 +1,22 @@
-const webpack = require('webpack');
-const { resolve } = require('path');
-const MiniCssExtractPlugin = require('mini-css-extract-plugin')
-const HtmlWebpackPlugin = require('html-webpack-plugin');
+const webpack = require("webpack");
+const {resolve} = require("path");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
 
-const isProd = process.env.NODE_ENV === 'production';
+const isProd = process.env.NODE_ENV === "production";
 
 const config = {
   devtool: "source-map",
-  mode: isProd ? 'production' : 'development',
+  mode: isProd ? "production" : "development",
   entry: {
-    index: './src/index.tsx',
+    index: "./src/index.tsx",
   },
   output: {
-    path: resolve(__dirname, 'dist'),
-    filename: '[name].js',
+    path: resolve(__dirname, "dist"),
+    filename: "[name].js",
   },
   resolve: {
-    extensions: ['.js', '.jsx', '.ts', '.tsx'],
+    extensions: [".js", ".jsx", ".ts", ".tsx"],
   },
   module: {
     rules: [
@@ -27,68 +27,72 @@ const config = {
         },
       },
       {
-      test: /\.scss$/,
-      use: [
+        test: /\.scss$/,
+        use: [
           MiniCssExtractPlugin.loader,
           {
-            loader: 'css-loader'
+            loader: "css-loader",
           },
           {
-            loader: 'sass-loader',
+            loader: "sass-loader",
             options: {
               sourceMap: true,
-            }
-          }
-        ]
-      },{
+            },
+          },
+        ],
+      },
+      {
         test: /\.tsx?$/,
-        use: 'babel-loader',
+        use: "babel-loader",
         exclude: /node_modules/,
-      }
+      },
+      {
+        use: "null-loader",
+        test: /@chainsafe\/hashtree/,
+      },
     ],
   },
   plugins: [
     new webpack.ProvidePlugin({
-      process: 'process/browser',
-      Buffer: ['buffer', 'Buffer'],
+      process: "process/browser",
+      Buffer: ["buffer", "Buffer"],
     }),
     new MiniCssExtractPlugin({
-      filename: 'css/[name].bundle.css'
+      filename: "css/[name].bundle.css",
     }),
     new HtmlWebpackPlugin({
-      title: 'Simple Serialize | Chainsafe Systems',
-      template: 'src/index.html',
+      title: "Simple Serialize | Chainsafe Systems",
+      template: "src/index.html",
     }),
   ],
 };
 
 if (isProd) {
   config.optimization = {
-    minimizer: [
-    ],
+    minimizer: [],
   };
 } else {
   config.devServer = {
     port: 8080, // https://webpack.js.org/configuration/dev-server/#devserverport
     open: true, // https://webpack.js.org/configuration/dev-server/#devserveropen
     hot: true, // https://webpack.js.org/configuration/dev-server/#devserverhot
     compress: true, // https://webpack.js.org/configuration/dev-server/#devservercompress
-    stats: 'errors-only', // https://webpack.js.org/configuration/dev-server/#devserverstats-
+    stats: "errors-only", // https://webpack.js.org/configuration/dev-server/#devserverstats-
     overlay: true, // https://webpack.js.org/configuration/dev-server/#devserveroverlay
   };
 }
 
 const workerConfig = {
   name: "worker",
   resolve: {
-    extensions: ['.js', '.jsx', '.ts', '.tsx'],
+    extensions: [".js", ".jsx", ".ts", ".tsx"],
   },
   entry: {
-    index: './src/components/worker/index.ts',
+    index: "./src/components/worker/index.ts",
   },
   output: {
-    path: resolve(__dirname, 'dist'),
-    filename: 'worker.js',
+    path: resolve(__dirname, "dist"),
+    filename: "worker.js",
   },
   module: {
     rules: [
@@ -100,21 +104,25 @@ const workerConfig = {
       },
       {
         test: /worker?$/,
-        loader: 'threads-webpack-plugin',
+        loader: "threads-webpack-plugin",
       },
       {
         test: /\.ts?$/,
-        use: 'babel-loader',
+        use: "babel-loader",
         exclude: /node_modules/,
-      }
+      },
+      {
+        use: "null-loader",
+        test: /@chainsafe\/hashtree/,
+      },
     ],
   },
   plugins: [
     new webpack.ProvidePlugin({
-      process: 'process/browser',
-      Buffer: ['buffer', 'Buffer'],
+      process: "process/browser",
+      Buffer: ["buffer", "Buffer"],
     }),
-  ]
-}
+  ],
+};
 
 module.exports = [config, workerConfig];

packages/ssz/src/index.ts

@@ -26,13 +26,21 @@ export {Type, ValueOf, JsonPath, ByteViews} from "./type/abstract";
 export {BasicType, isBasicType} from "./type/basic";
 export {CompositeType, CompositeTypeAny, CompositeView, CompositeViewDU, isCompositeType} from "./type/composite";
 export {TreeView} from "./view/abstract";
-export {ValueOfFields} from "./view/container";
+export {ValueOfFields, ContainerTypeGeneric} from "./view/container";
 export {TreeViewDU} from "./viewDU/abstract";
+export {ListCompositeTreeViewDU} from "./viewDU/listComposite";
+export {ListBasicTreeViewDU} from "./viewDU/listBasic";
+export {ArrayCompositeTreeViewDUCache} from "./viewDU/arrayComposite";
+export {ContainerNodeStructTreeViewDU} from "./viewDU/containerNodeStruct";
 
 // Values
 export {BitArray, getUint8ByteToBitBooleanArray} from "./value/bitArray";
 
 // Utils
 export {fromHexString, toHexString, byteArrayEquals} from "./util/byteArray";
+export {ReusableListIterator} from "./util/reusableListIterator";
 
 export {hash64, symbolCachedPermanentRoot} from "./util/merkleize";
+
+// others
+export {BranchNodeStruct} from "./branchNodeStruct";

packages/ssz/src/interface.ts

@@ -16,6 +16,12 @@ export interface List<T> extends ArrayLike<T> {
   pop(): T | undefined;
 }
 
+export interface ListIterator<T> {
+  readonly length: number;
+  push(...values: T[]): void;
+  [Symbol.iterator](): Iterator<T>;
+}
+
 export type Container<T extends Record<string, unknown>> = T;
 
 export type ByteVector = Vector<number>;

packages/ssz/src/type/abstract.ts

@@ -145,6 +145,11 @@ export abstract class Type<V> {
    */
   abstract hashTreeRoot(value: V): Uint8Array;
 
+  /**
+   * Same to hashTreeRoot() but here we write result to output.
+   */
+  abstract hashTreeRootInto(value: V, output: Uint8Array, offset: number): void;
+
   // JSON support
 
   /** Parse JSON representation of a type to value */

packages/ssz/src/type/arrayComposite.ts

@@ -211,21 +211,29 @@ export function tree_deserializeFromBytesArrayComposite<ElementType extends Comp
   }
 }
 
-/**
- * @param length In List length = value.length, Vector length = fixed value
- */
-export function value_getRootsArrayComposite<ElementType extends CompositeType<unknown, unknown, unknown>>(
+export function value_getChunkBytesArrayComposite<ElementType extends CompositeType<unknown, unknown, unknown>>(
   elementType: ElementType,
   length: number,
-  value: ValueOf<ElementType>[]
-): Uint8Array[] {
-  const roots = new Array<Uint8Array>(length);
+  value: ValueOf<ElementType>[],
+  chunkBytesBuffer: Uint8Array
+): Uint8Array {
+  const isOddChunk = length % 2 === 1;
+  const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32;
+  if (chunkBytesLen > chunkBytesBuffer.length) {
+    throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`);
+  }
+  const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen);
 
   for (let i = 0; i < length; i++) {
-    roots[i] = elementType.hashTreeRoot(value[i]);
+    elementType.hashTreeRootInto(value[i], chunkBytes, i * 32);
+  }
+
+  if (isOddChunk) {
+    // similar to append zeroHash(0)
+    chunkBytes.subarray(length * 32, chunkBytesLen).fill(0);
   }
 
-  return roots;
+  return chunkBytes;
 }
 
 function readOffsetsArrayComposite(

packages/ssz/src/type/basic.ts

@@ -30,11 +30,17 @@ export abstract class BasicType<V> extends Type<V> {
   }
 
   hashTreeRoot(value: V): Uint8Array {
-    // TODO: Optimize
-    const uint8Array = new Uint8Array(32);
+    const root = new Uint8Array(32);
+    this.hashTreeRootInto(value, root, 0);
+    return root;
+  }
+
+  hashTreeRootInto(value: V, output: Uint8Array, offset: number): void {
+    const uint8Array = output.subarray(offset, offset + 32);
+    // output could have preallocated data, some types may not fill the whole 32 bytes
+    uint8Array.fill(0);
     const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
     this.value_serializeToBytes({uint8Array, dataView}, 0, value);
-    return uint8Array;
   }
 
   clone(value: V): V {

packages/ssz/src/type/bitArray.ts

@@ -1,10 +1,10 @@
 import {concatGindices, Gindex, Node, toGindex, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray";
-import {splitIntoRootChunks} from "../util/merkleize";
 import {CompositeType, LENGTH_GINDEX} from "./composite";
 import {BitArray} from "../value/bitArray";
 import {BitArrayTreeView} from "../view/bitArray";
 import {BitArrayTreeViewDU} from "../viewDU/bitArray";
+import {getChunkBytes} from "./byteArray";
 
 /* eslint-disable @typescript-eslint/member-ordering */
 
@@ -40,8 +40,15 @@ export abstract class BitArrayType extends CompositeType<BitArray, BitArrayTreeV
 
   // Merkleization
 
-  protected getRoots(value: BitArray): Uint8Array[] {
-    return splitIntoRootChunks(value.uint8Array);
+  protected getChunkBytes(value: BitArray): Uint8Array {
+    // reallocate this.merkleBytes if needed
+    if (value.uint8Array.length > this.chunkBytesBuffer.length) {
+      const chunkCount = Math.ceil(value.bitLen / 8 / 32);
+      const chunkBytes = chunkCount * 32;
+      // pad 1 chunk if maxChunkCount is not even
+      this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes);
+    }
+    return getChunkBytes(value.uint8Array, this.chunkBytesBuffer);
   }
 
   // Proofs

packages/ssz/src/type/bitList.ts

@@ -1,5 +1,11 @@
-import {getNodesAtDepth, Node, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree";
-import {mixInLength, maxChunksToDepth} from "../util/merkleize";
+import {
+  getNodesAtDepth,
+  merkleizeInto,
+  Node,
+  packedNodeRootsToBytes,
+  packedRootsBytesToNode,
+} from "@chainsafe/persistent-merkle-tree";
+import {maxChunksToDepth} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
 import {ByteViews} from "./composite";
@@ -29,6 +35,12 @@ export class BitListType extends BitArrayType {
   readonly maxSize: number;
   readonly maxChunkCount: number;
   readonly isList = true;
+  readonly mixInLengthChunkBytes = new Uint8Array(64);
+  readonly mixInLengthBuffer = Buffer.from(
+    this.mixInLengthChunkBytes.buffer,
+    this.mixInLengthChunkBytes.byteOffset,
+    this.mixInLengthChunkBytes.byteLength
+  );
 
   constructor(readonly limitBits: number, opts?: BitListOptions) {
     super();
@@ -101,7 +113,18 @@ export class BitListType extends BitArrayType {
   // Merkleization: inherited from BitArrayType
 
   hashTreeRoot(value: BitArray): Uint8Array {
-    return mixInLength(super.hashTreeRoot(value), value.bitLen);
+    const root = new Uint8Array(32);
+    this.hashTreeRootInto(value, root, 0);
+    return root;
+  }
+
+  hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void {
+    super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0);
+    // mixInLength
+    this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6);
+    // one for hashTreeRoot(value), one for length
+    const chunkCount = 2;
+    merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
   }
 
   // Proofs: inherited from BitArrayType