Commitment: partial unfold

.github/workflows/ci.yml

@@ -6,8 +6,7 @@ on:
       - 'release/**'
   pull_request:
     branches:
-      - main
-      - 'release/**'
+      - '**'
     types:
       - opened
       - reopened

cmd/commitment-prefix/main.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -42,6 +43,8 @@ var (
 	flagConcurrency     = flag.Int("j", 4, "amount of concurrently proceeded files")
 	flagTrieVariant     = flag.String("trie", "hex", "commitment trie variant (values are hex and bin)")
 	flagCompression     = flag.String("compression", "none", "compression type (none, k, v, kv)")
+	flagPrintState      = flag.Bool("state", false, "print state of file")
+	flagDepth           = flag.Int("depth", 0, "depth of the prefixes to analyze")
 )
 
 func main() {
@@ -69,9 +72,10 @@ func proceedFiles(files []string) {
 
 	for i, fp := range files {
 		fpath, pos := fp, i
+		_ = pos
 		<-sema
 
-		fmt.Printf("\r[%d/%d] - %s..", pos+1, len(files), path.Base(fpath))
+		fmt.Printf("[%d/%d] - %s..", pos+1, len(files), path.Base(fpath))
 
 		wg.Add(1)
 		go func(wg *sync.WaitGroup, mu *sync.Mutex) {
@@ -86,16 +90,19 @@ func proceedFiles(files []string) {
 
 			mu.Lock()
 			page.AddCharts(
-
 				prefixLenCountChart(fpath, stat),
 				countersChart(fpath, stat),
+				mediansChart(fpath, stat),
 				fileContentsMapChart(fpath, stat),
 			)
 			mu.Unlock()
 		}(&wg, &mu)
 	}
 	wg.Wait()
 	fmt.Println()
+	if *flagPrintState {
+		return
+	}
 
 	dir := filepath.Dir(files[0])
 	if *flagOutputDirectory != "" {
@@ -180,21 +187,39 @@ func extractKVPairFromCompressed(filename string, keysSink chan commitment.Branc
 	size := dec.Size()
 	paris := dec.Count() / 2
 	cpair := 0
-
+	depth := *flagDepth
+	var afterValPos uint64
+	var key, val []byte
 	getter := seg.NewReader(dec.MakeGetter(), fc)
 	for getter.HasNext() {
-		key, _ := getter.Next(nil)
+		key, _ = getter.Next(key[:0])
 		if !getter.HasNext() {
 			return errors.New("invalid key/value pair during decompression")
 		}
-		val, afterValPos := getter.Next(nil)
+		if *flagPrintState && !bytes.Equal(key, []byte("state")) {
+			getter.Skip()
+			continue
+		}
+
+		val, afterValPos = getter.Next(val[:0])
 		cpair++
+		if bytes.Equal(key, []byte("state")) {
+			str, err := commitment.HexTrieStateToString(val)
+			if err != nil {
+				fmt.Printf("[ERR] failed to decode state: %v", err)
+			}
+			fmt.Printf("\n%s: %s\n", dec.FileName(), str)
+			continue
+		}
 
 		if cpair%100000 == 0 {
 			fmt.Printf("\r%s pair %d/%d %s/%s", filename, cpair, paris,
 				datasize.ByteSize(afterValPos).HumanReadable(), datasize.ByteSize(size).HumanReadable())
 		}
 
+		if depth > len(key) {
+			continue
+		}
 		stat := commitment.DecodeBranchAndCollectStat(key, val, tv)
 		if stat == nil {
 			fmt.Printf("failed to decode branch: %x %x\n", key, val)
@@ -252,7 +277,7 @@ func prefixLenCountChart(fname string, data *overallStat) *charts.Pie {
 	pie := charts.NewPie()
 	pie.SetGlobalOptions(
 		charts.WithTooltipOpts(opts.Tooltip{Show: true}),
-		charts.WithTitleOpts(opts.Title{Subtitle: fname, Title: "key prefix length distribution (bytes)", Top: "25"}),
+		charts.WithTitleOpts(opts.Title{Subtitle: filepath.Base(fname), Title: "key prefix length distribution (bytes)", Top: "25"}),
 	)
 
 	pie.AddSeries("prefixLen/count", items)
@@ -285,13 +310,17 @@ func fileContentsMapChart(fileName string, data *overallStat) *charts.TreeMap {
 			Value: int(data.branches.ExtSize),
 		},
 		{
-			Name:  "apk",
+			Name:  "accountKey",
 			Value: int(data.branches.APKSize),
 		},
 		{
-			Name:  "spk",
+			Name:  "storageKey",
 			Value: int(data.branches.SPKSize),
 		},
+		{
+			Name:  "leafHashes",
+			Value: int(data.branches.LeafHashSize),
+		},
 	}
 
 	graph := charts.NewTreeMap()
@@ -305,12 +334,10 @@ func fileContentsMapChart(fileName string, data *overallStat) *charts.TreeMap {
 	)
 
 	// Add initialized data to graph.
-	graph.AddSeries(fileName, TreeMap).
+	graph.AddSeries(filepath.Base(fileName), TreeMap).
 		SetSeriesOptions(
 			charts.WithTreeMapOpts(
 				opts.TreeMapChart{
-					Animation: true,
-					//Roam:       true,
 					UpperLabel: &opts.UpperLabel{Show: true, Color: "#fff"},
 					Levels: &[]opts.TreeMapLevel{
 						{ // Series
@@ -383,9 +410,6 @@ func countersChart(fname string, data *overallStat) *charts.Sankey {
 	sankey.SetGlobalOptions(
 		charts.WithLegendOpts(opts.Legend{Show: true}),
 		charts.WithTooltipOpts(opts.Tooltip{Show: true}),
-		//charts.WithTitleOpts(opts.Title{
-		//	Title: "Sankey-basic-example",
-		//}),
 	)
 
 	nodes := []opts.SankeyNode{
@@ -394,15 +418,53 @@ func countersChart(fname string, data *overallStat) *charts.Sankey {
 		{Name: "SPK"},
 		{Name: "Hashes"},
 		{Name: "Extensions"},
+		{Name: "LeafHashes"},
 	}
 	sankeyLink := []opts.SankeyLink{
 		{Source: nodes[0].Name, Target: nodes[1].Name, Value: float32(data.branches.APKCount)},
 		{Source: nodes[0].Name, Target: nodes[2].Name, Value: float32(data.branches.SPKCount)},
 		{Source: nodes[0].Name, Target: nodes[3].Name, Value: float32(data.branches.HashCount)},
 		{Source: nodes[0].Name, Target: nodes[4].Name, Value: float32(data.branches.ExtCount)},
+		{Source: nodes[0].Name, Target: nodes[5].Name, Value: float32(data.branches.LeafHashCount)},
+	}
+
+	sankey.AddSeries("Counts "+filepath.Base(fname), nodes, sankeyLink).
+		SetSeriesOptions(
+			charts.WithLineStyleOpts(opts.LineStyle{
+				Color:     "source",
+				Curveness: 0.5,
+			}),
+			charts.WithLabelOpts(opts.Label{
+				Show: true,
+			}),
+		)
+	return sankey
+}
+
+func mediansChart(fname string, data *overallStat) *charts.Sankey {
+	sankey := charts.NewSankey()
+	sankey.SetGlobalOptions(
+		charts.WithLegendOpts(opts.Legend{Show: true}),
+		charts.WithTooltipOpts(opts.Tooltip{Show: true}),
+	)
+
+	nodes := []opts.SankeyNode{
+		{Name: "Cells"},
+		{Name: "Addr"},
+		{Name: "Addr+Storage"},
+		{Name: "Hashes"},
+		{Name: "Extensions"},
+		{Name: "LeafHashes"},
+	}
+	sankeyLink := []opts.SankeyLink{
+		{Source: nodes[0].Name, Target: nodes[1].Name, Value: float32(data.branches.MedianAPK)},
+		{Source: nodes[0].Name, Target: nodes[2].Name, Value: float32(data.branches.MedianSPK)},
+		{Source: nodes[0].Name, Target: nodes[3].Name, Value: float32(data.branches.MedianHash)},
+		{Source: nodes[0].Name, Target: nodes[4].Name, Value: float32(data.branches.MedianExt)},
+		{Source: nodes[0].Name, Target: nodes[5].Name, Value: float32(data.branches.MedianLH)},
 	}
 
-	sankey.AddSeries(fname, nodes, sankeyLink).
+	sankey.AddSeries("Medians "+filepath.Base(fname), nodes, sankeyLink).
 		SetSeriesOptions(
 			charts.WithLineStyleOpts(opts.LineStyle{
 				Color:     "source",

cmd/integration/commands/stages.go

@@ -254,7 +254,7 @@ var cmdStageCustomTrace = &cobra.Command{
 }
 
 var cmdStagePatriciaTrie = &cobra.Command{
-	Use:   "rebuild_trie3_files",
+	Use:   "commitment_rebuild",
 	Short: "",
 	Run: func(cmd *cobra.Command, args []string) {
 		logger := debug.SetupCobra(cmd, "integration")
@@ -1178,20 +1178,15 @@ func stagePatriciaTrie(db kv.RwDB, ctx context.Context, logger log.Logger) error
 	if reset {
 		return reset2.Reset(ctx, db, stages.Execution)
 	}
-	tx, err := db.BeginRw(ctx)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
 
 	br, _ := blocksIO(db, logger)
 	historyV3 := true
 	cfg := stagedsync.StageTrieCfg(db, true /* checkRoot */, true /* saveHashesToDb */, false /* badBlockHalt */, dirs.Tmp, br, nil /* hd */, historyV3, agg)
 
-	if _, err := stagedsync.RebuildPatriciaTrieBasedOnFiles(tx, cfg, ctx, logger); err != nil {
+	if _, err := stagedsync.RebuildPatriciaTrieBasedOnFiles(ctx, cfg); err != nil {
 		return err
 	}
-	return tx.Commit()
+	return nil
 }
 
 func stageTxLookup(db kv.RwDB, ctx context.Context, logger log.Logger) error {