Use a binary file for genesis [#3397]

cmd/accumulated/cmd_init.go

@@ -33,6 +33,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/logging"
 	cfg "gitlab.com/accumulatenetwork/accumulate/internal/node/config"
 	accumulated "gitlab.com/accumulatenetwork/accumulate/internal/node/daemon"
+	"gitlab.com/accumulatenetwork/accumulate/internal/node/genesis"
 	client "gitlab.com/accumulatenetwork/accumulate/pkg/client/api/v2"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/proxy"
 	"gitlab.com/accumulatenetwork/accumulate/protocol"
@@ -679,7 +680,12 @@ func initNode(cmd *cobra.Command, args []string) (string, error) {
 		return "", fmt.Errorf("load/generate node key files, %v", err)
 	}
 
-	err = accumulated.WriteNodeFiles(config, privValKey, nodeKey, genDoc)
+	config.Genesis = "config/genesis.snap"
+	genDocBytes, err := genesis.ConvertJsonToSnapshot(genDoc)
+	if err != nil {
+		return "", fmt.Errorf("write node files, %v", err)
+	}
+	err = accumulated.WriteNodeFiles(config, privValKey, nodeKey, genDocBytes)
 	if err != nil {
 		return "", fmt.Errorf("write node files, %v", err)
 	}

cmd/accumulated/cmd_init_network.go

@@ -141,6 +141,9 @@ func initNetworkLocalFS(cmd *cobra.Command, netInit *accumulated.NetworkInit) {
 	for _, configs := range configs {
 		for _, configs := range configs {
 			for _, config := range configs {
+				// Use binary genesis files
+				config.Genesis = "config/genesis.snap"
+
 				if flagInit.LogLevels != "" {
 					config.LogLevel = flagInit.LogLevels
 				}

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/FactomProject/factomd v1.13.0
 	github.com/chzyer/readline v1.5.1
 	github.com/cometbft/cometbft v0.38.0-rc3
+	github.com/cosmos/gogoproto v1.4.11
 	github.com/ethereum/go-ethereum v1.12.2
 	github.com/fatih/astrewrite v0.0.0-20191207154002-9094e544fcef
 	github.com/gobeam/stringy v0.0.6
@@ -85,7 +86,6 @@ require (
 	github.com/chigopher/pathlib v0.12.0 // indirect
 	github.com/cometbft/cometbft-db v0.7.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
-	github.com/cosmos/gogoproto v1.4.11 // indirect
 	github.com/curioswitch/go-reassign v0.2.0 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect

internal/core/execute/v1/simulator/simulator.go

@@ -11,7 +11,6 @@ package simulator
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"sync"
@@ -324,9 +323,7 @@ func (s *Simulator) InitFromGenesisWith(values *core.GlobalValues) {
 
 	// Execute bootstrap after the entire network is known
 	for _, x := range s.Executors {
-		var snap []byte
-		require.NoError(s, json.Unmarshal(genDocs[x.Partition.Id].AppState, &snap))
-		require.NoError(s, snapshot.FullRestore(x.Database, ioutil2.NewBuffer(snap), x.Executor.Logger, x.Executor.Describe.PartitionUrl()))
+		require.NoError(s, snapshot.FullRestore(x.Database, ioutil2.NewBuffer(genDocs[x.Partition.Id]), x.Executor.Logger, x.Executor.Describe.PartitionUrl()))
 		require.NoError(s, x.Executor.Init(x.Database))
 	}
 }

internal/database/snapshot.go

@@ -16,6 +16,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/database/record"
 	"gitlab.com/accumulatenetwork/accumulate/internal/database/smt/storage"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database"
+	kvb "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/badger"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database/snapshot"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database/values"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
@@ -29,8 +30,9 @@ const collectIndexTxnPrefix = "txn."
 const collectIndexRecordPrefix = "rec."
 
 type CollectOptions struct {
-	BuildIndex bool
-	Predicate  func(database.Record) (bool, error)
+	BuildIndex     bool
+	Predicate      func(database.Record) (bool, error)
+	DidWriteHeader func(*snapshot.Writer) error
 
 	Metrics *CollectMetrics
 }
@@ -59,6 +61,13 @@ func (db *Database) Collect(file io.WriteSeeker, partition *url.URL, opts *Colle
 		return errors.UnknownError.Wrap(err)
 	}
 
+	if opts.DidWriteHeader != nil {
+		err = opts.DidWriteHeader(w)
+		if err != nil {
+			return errors.UnknownError.Wrap(err)
+		}
+	}
+
 	// Collect the BPT
 	err = db.collectBPT(w, opts)
 	if err != nil {
@@ -74,7 +83,7 @@ func (db *Database) Collect(file io.WriteSeeker, partition *url.URL, opts *Colle
 		_ = os.RemoveAll(dir)
 	}(dir)
 
-	index, err := badger.Open(badger.DefaultOptions(dir))
+	index, err := badger.Open(badger.DefaultOptions(dir).WithLogger(kvb.Slogger{}))
 	if err != nil {
 		return errors.UnknownError.Wrap(err)
 	}
@@ -226,9 +235,10 @@ func (db *Database) collectMessages(w *snapshot.Writer, index *badger.DB, opts *
 			return errors.UnknownError.WithFormat("collect %x: %w", hash, err)
 		}
 
-		// Collect the transaction status (which is the only part of the
-		// transaction entity that is still used by exec v2)
-		err = records.Collect(batch.newTransaction(transactionKey{Hash: hash}).newStatus(), copts)
+		// Collect the transaction's records. Executor v2 only uses the
+		// transaction status, but transactions and signatures from v1 are still
+		// stored here, so they should be collected.
+		err = records.Collect(batch.newTransaction(transactionKey{Hash: hash}), copts)
 		if err != nil {
 			return errors.UnknownError.WithFormat("collect %x status: %w", hash, err)
 		}

internal/node/abci/accumulator.go

@@ -21,7 +21,6 @@ import (
 	"github.com/cometbft/cometbft/crypto"
 	"github.com/cometbft/cometbft/libs/log"
 	protocrypto "github.com/cometbft/cometbft/proto/tendermint/crypto"
-	"github.com/cometbft/cometbft/types"
 	"github.com/cometbft/cometbft/version"
 	"gitlab.com/accumulatenetwork/accumulate"
 	"gitlab.com/accumulatenetwork/accumulate/exp/ioutil"
@@ -32,6 +31,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/database/snapshot"
 	"gitlab.com/accumulatenetwork/accumulate/internal/logging"
 	"gitlab.com/accumulatenetwork/accumulate/internal/node/config"
+	"gitlab.com/accumulatenetwork/accumulate/internal/node/genesis"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/types/messaging"
 	"gitlab.com/accumulatenetwork/accumulate/protocol"
@@ -242,7 +242,7 @@ func (app *Accumulator) Info(context.Context, *abci.RequestInfo) (*abci.Response
 	}
 
 	// Check the genesis document
-	genDoc, err := types.GenesisDocFromFile(app.Config.GenesisFile())
+	genDoc, err := genesis.DocProvider(app.Config)()
 	if err != nil {
 		return nil, err
 	}

internal/node/daemon/init.go

@@ -8,7 +8,6 @@ package accumulated
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -20,7 +19,6 @@ import (
 
 	"github.com/cometbft/cometbft/crypto/ed25519"
 	tmed25519 "github.com/cometbft/cometbft/crypto/ed25519"
-	tmbytes "github.com/cometbft/cometbft/libs/bytes"
 	tmjson "github.com/cometbft/cometbft/libs/json"
 	"github.com/cometbft/cometbft/libs/log"
 	tmos "github.com/cometbft/cometbft/libs/os"
@@ -210,83 +208,34 @@ func ConfigureNodePorts(node *NodeInit, cfg *config.Config, part protocol.Partit
 	cfg.Accumulate.API.ListenAddress = node.Listen().Scheme("http").PartitionType(part).AccumulateAPI().String()
 }
 
-func BuildGenesisDocs(network *NetworkInit, globals *core.GlobalValues, time time.Time, logger log.Logger, factomAddresses func() (io.Reader, error), snapshots []func() (ioutil2.SectionReader, error)) (map[string]*tmtypes.GenesisDoc, error) {
-	docs := map[string]*tmtypes.GenesisDoc{}
+func BuildGenesisDocs(network *NetworkInit, globals *core.GlobalValues, time time.Time, logger log.Logger, factomAddresses func() (io.Reader, error), snapshots []func() (ioutil2.SectionReader, error)) (map[string][]byte, error) {
+	docs := map[string][]byte{}
 	var operators [][]byte
 	netinfo := new(protocol.NetworkDefinition)
 	netinfo.NetworkName = network.Id
 	netinfo.AddPartition(protocol.Directory, protocol.PartitionTypeDirectory)
 
-	var dnTmValidators []tmtypes.GenesisValidator
-
 	var i int
 	for _, bvn := range network.Bvns {
-		var bvnTmValidators []tmtypes.GenesisValidator
+		netinfo.AddPartition(bvn.Id, protocol.PartitionTypeBlockValidator)
 
-		for j, node := range bvn.Nodes {
+		for _, node := range bvn.Nodes {
 			i++
 			key := tmed25519.PrivKey(node.PrivValKey)
 			operators = append(operators, key.PubKey().Bytes())
-
 			netinfo.AddValidator(key.PubKey().Bytes(), protocol.Directory, node.DnnType == config.Validator)
 			netinfo.AddValidator(key.PubKey().Bytes(), bvn.Id, node.BvnnType == config.Validator)
-
-			if node.DnnType == config.Validator {
-				dnTmValidators = append(dnTmValidators, tmtypes.GenesisValidator{
-					Name:    fmt.Sprintf("Directory.%d", i),
-					Address: key.PubKey().Address(),
-					PubKey:  key.PubKey(),
-					Power:   1,
-				})
-			}
-
-			if node.BvnnType == config.Validator {
-				bvnTmValidators = append(bvnTmValidators, tmtypes.GenesisValidator{
-					Name:    fmt.Sprintf("%s.%d", bvn.Id, j+1),
-					Address: key.PubKey().Address(),
-					PubKey:  key.PubKey(),
-					Power:   1,
-				})
-			}
-		}
-
-		netinfo.AddPartition(bvn.Id, protocol.PartitionTypeBlockValidator)
-		docs[bvn.Id] = &tmtypes.GenesisDoc{
-			ChainID:         bvn.Id,
-			GenesisTime:     time,
-			InitialHeight:   protocol.GenesisBlock + 1,
-			Validators:      bvnTmValidators,
-			ConsensusParams: tmtypes.DefaultConsensusParams(),
 		}
 	}
 
-	var bsnTmValidators []tmtypes.GenesisValidator
 	if network.Bsn != nil {
-		for j, node := range network.Bsn.Nodes {
+		for _, node := range network.Bsn.Nodes {
 			key := tmed25519.PrivKey(node.PrivValKey)
 			operators = append(operators, key.PubKey().Bytes())
-
 			netinfo.AddValidator(key.PubKey().Bytes(), network.Bsn.Id, node.BsnnType == config.Validator)
-
-			if node.BsnnType == config.Validator {
-				bsnTmValidators = append(bsnTmValidators, tmtypes.GenesisValidator{
-					Name:    fmt.Sprintf("%s.%d", network.Bsn.Id, j+1),
-					Address: key.PubKey().Address(),
-					PubKey:  key.PubKey(),
-					Power:   1,
-				})
-			}
 		}
 	}
 
-	docs[protocol.Directory] = &tmtypes.GenesisDoc{
-		ChainID:         protocol.Directory,
-		GenesisTime:     time,
-		InitialHeight:   protocol.GenesisBlock + 1,
-		Validators:      dnTmValidators,
-		ConsensusParams: tmtypes.DefaultConsensusParams(),
-	}
-
 	globals.Network = netinfo
 
 	ids := []string{protocol.Directory}
@@ -314,7 +263,8 @@ func BuildGenesisDocs(network *NetworkInit, globals *core.GlobalValues, time tim
 			netType = protocol.PartitionTypeDirectory
 		}
 		snapBuf := new(ioutil2.Buffer)
-		root, err := genesis.Init(snapBuf, genesis.InitOpts{
+		err = genesis.Init(snapBuf, genesis.InitOpts{
+			NetworkID:       network.Id,
 			PartitionId:     id,
 			NetworkType:     netType,
 			GenesisTime:     time,
@@ -323,6 +273,7 @@ func BuildGenesisDocs(network *NetworkInit, globals *core.GlobalValues, time tim
 			OperatorKeys:    operators,
 			FactomAddresses: factomAddresses,
 			Snapshots:       snapshots,
+			ConsensusParams: tmtypes.DefaultConsensusParams(),
 		})
 		if err != nil {
 			return nil, err
@@ -344,33 +295,17 @@ func BuildGenesisDocs(network *NetworkInit, globals *core.GlobalValues, time tim
 			}
 		}
 
-		docs[id].AppHash = root
-		docs[id].AppState, err = json.Marshal(snapBuf.Bytes())
-		if err != nil {
-			return nil, err
-		}
+		docs[id] = snapBuf.Bytes()
 	}
 
 	if network.Bsn != nil {
-		b, err := json.Marshal(bsnSnapBuf.Bytes())
-		if err != nil {
-			return nil, err
-		}
-		docs[network.Bsn.Id] = &tmtypes.GenesisDoc{
-			ChainID:         network.Bsn.Id,
-			GenesisTime:     time,
-			InitialHeight:   1,
-			Validators:      bsnTmValidators,
-			ConsensusParams: tmtypes.DefaultConsensusParams(),
-			AppHash:         make(tmbytes.HexBytes, 32),
-			AppState:        b,
-		}
+		docs[network.Bsn.Id] = bsnSnapBuf.Bytes()
 	}
 
 	return docs, nil
 }
 
-func WriteNodeFiles(cfg *config.Config, privValKey, nodeKey []byte, genDoc *tmtypes.GenesisDoc) (err error) {
+func WriteNodeFiles(cfg *config.Config, privValKey, nodeKey []byte, genDoc []byte) (err error) {
 	defer func() {
 		if err != nil {
 			_ = os.RemoveAll(cfg.RootDir)
@@ -425,7 +360,7 @@ func WriteNodeFiles(cfg *config.Config, privValKey, nodeKey []byte, genDoc *tmty
 		return fmt.Errorf("failed to write node key: %w", err)
 	}
 
-	err = genDoc.SaveAs(cfg.GenesisFile())
+	err = os.WriteFile(cfg.GenesisFile(), genDoc, 0644)
 	if err != nil {
 		return fmt.Errorf("failed to write genesis file: %w", err)
 	}

internal/node/daemon/run.go

@@ -51,6 +51,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/node/config"
 	"gitlab.com/accumulatenetwork/accumulate/internal/node/connections"
 	statuschk "gitlab.com/accumulatenetwork/accumulate/internal/node/connections/status"
+	"gitlab.com/accumulatenetwork/accumulate/internal/node/genesis"
 	nodeapi "gitlab.com/accumulatenetwork/accumulate/internal/node/http"
 	v3 "gitlab.com/accumulatenetwork/accumulate/pkg/api/v3"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/api/v3/message"
@@ -379,7 +380,7 @@ func (d *Daemon) startConsensus(app types.Application) error {
 		d.privVal,
 		d.nodeKey,
 		proxy.NewLocalClientCreator(app),
-		tmnode.DefaultGenesisDocProviderFunc(&d.Config.Config),
+		genesis.DocProvider(d.Config),
 		tmcfg.DefaultDBProvider,
 		tmnode.DefaultMetricsProvider(d.Config.Instrumentation),
 		d.Logger,
@@ -390,7 +391,6 @@ func (d *Daemon) startConsensus(app types.Application) error {
 	d.node = &node.Node{Node: tmn, Config: d.Config, ABCI: app}
 
 	// Start node
-	// TODO Feed Tendermint logger to service logger
 	err = d.node.Start()
 	if err != nil {
 		return errors.UnknownError.WithFormat("start consensus: %v", err)