store merkle tree in sql database

cmd/explored/node.go

@@ -17,7 +17,6 @@ import (
 	"go.sia.tech/coreutils/chain"
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/explored/explorer"
-	"go.sia.tech/explored/internal/explorerutil"
 	"go.sia.tech/explored/internal/syncerutil"
 	"go.sia.tech/explored/persist/sqlite"
 	"go.uber.org/zap"
@@ -178,13 +177,8 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo
 	if err := os.MkdirAll(hashPath, fs.ModePerm); err != nil {
 		return nil, err
 	}
-	hashStore, err := explorerutil.NewHashStore(hashPath)
-	if err != nil {
-		return nil, err
-	}
-	cm.AddSubscriber(hashStore, tip)
 
-	e := explorer.NewExplorer(store, hashStore)
+	e := explorer.NewExplorer(store)
 
 	l, err := net.Listen("tcp", addr)
 	if err != nil {
@@ -248,7 +242,6 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, logger *zap.Lo
 				l.Close()
 				<-ch
 				db.Close()
-				hashStore.Commit()
 			}
 		},
 	}, nil

explorer/explorer.go

@@ -5,15 +5,6 @@ import (
 	"go.sia.tech/coreutils/chain"
 )
 
-// A HashStore stores the state element merkle tree.
-type HashStore interface {
-	chain.Subscriber
-
-	Commit() error
-	MerkleProof(leafIndex uint64) ([]types.Hash256, error)
-	ModifyLeaf(elem types.StateElement) error
-}
-
 // A Store is a database that stores information about elements, contracts,
 // and blocks.
 type Store interface {
@@ -26,22 +17,23 @@ type Store interface {
 	UnspentSiacoinOutputs(address types.Address, limit, offset uint64) ([]SiacoinOutput, error)
 	UnspentSiafundOutputs(address types.Address, limit, offset uint64) ([]SiafundOutput, error)
 	Balance(address types.Address) (sc types.Currency, sf uint64, err error)
+
+	MerkleProof(leafIndex uint64) ([]types.Hash256, error)
 }
 
 // Explorer implements a Sia explorer.
 type Explorer struct {
-	s  Store
-	hs HashStore
+	s Store
 }
 
 // NewExplorer returns a Sia explorer.
-func NewExplorer(s Store, hs HashStore) *Explorer {
-	return &Explorer{s: s, hs: hs}
+func NewExplorer(s Store) *Explorer {
+	return &Explorer{s: s}
 }
 
 // MerkleProof gets the merkle proof with the given leaf index.
 func (e *Explorer) MerkleProof(leafIndex uint64) ([]types.Hash256, error) {
-	return e.hs.MerkleProof(leafIndex)
+	return e.s.MerkleProof(leafIndex)
 }
 
 // Tip returns the tip of the best known valid chain.

persist/sqlite/consensus.go

@@ -163,6 +163,7 @@ func (s *Store) addTransactions(dbTxn txn, bid types.BlockID, txns []types.Trans
 type consensusUpdate interface {
 	ForEachSiacoinElement(fn func(sce types.SiacoinElement, spent bool))
 	ForEachSiafundElement(fn func(sfe types.SiafundElement, spent bool))
+	ForEachFileContractElement(fn func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool))
 }
 
 func (s *Store) updateBalances(dbTxn txn, update consensusUpdate) error {
@@ -372,6 +373,10 @@ func (s *Store) applyUpdates() error {
 			} else if err := s.addTransactions(dbTxn, update.Block.ID(), update.Block.Transactions, scDBIds, sfDBIds); err != nil {
 				return fmt.Errorf("applyUpdates: failed to add transactions: addTransactions: %w", err)
 			}
+
+			if err := s.updateLeaves(dbTxn, update); err != nil {
+				return err
+			}
 		}
 		s.pendingUpdates = s.pendingUpdates[:0]
 		return nil
@@ -389,7 +394,8 @@ func (s *Store) revertUpdate(cru *chain.RevertUpdate) error {
 		} else if err := s.updateBalances(dbTxn, cru); err != nil {
 			return fmt.Errorf("revertUpdate: failed to update balances: %w", err)
 		}
-		return nil
+
+		return s.updateLeaves(dbTxn, cru)
 	})
 }
 

persist/sqlite/init.go

@@ -69,6 +69,9 @@ func (s *Store) init() error {
 		return fmt.Errorf("failed to disable foreign key constraints: %w", err)
 	}
 
+	// error is ignored -- the database may not have been initialized yet.
+	s.db.QueryRow("SELECT COUNT(*) FROM merkle_proofs WHERE i = 0").Scan(&s.numLeaves)
+
 	version := getDBVersion(s.db)
 	switch {
 	case version == 0:

persist/sqlite/init.sql

@@ -123,5 +123,12 @@ CREATE TABLE transaction_siafund_outputs (
 
 CREATE INDEX transaction_siafund_outputs_transaction_id_index ON transaction_siafund_outputs(transaction_id);
 
+CREATE TABLE merkle_proofs (
+        i INTEGER NOT NULL,
+        j INTEGER NOT NULL,
+        hash BLOB NOT NULL,
+        PRIMARY KEY(i ,j)
+);
+
 -- initialize the global settings table
 INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed

persist/sqlite/merkle.go

@@ -0,0 +1,96 @@
+package sqlite
+
+import (
+	"math/bits"
+
+	"go.sia.tech/core/types"
+)
+
+func (s *Store) updateLeaves(dbTxn txn, update consensusUpdate) error {
+	modifyLeaf := func(stmt *loggedStmt, elem types.StateElement) error {
+		pos := elem.LeafIndex
+		for i, h := range elem.MerkleProof {
+			subtreeSize := uint64(1 << i)
+			if elem.LeafIndex&(1<<i) == 0 {
+				pos += subtreeSize
+			} else {
+				pos -= subtreeSize
+			}
+			// write hash h to (i, pos/subtreeSize)
+			encoded := dbEncode(h)
+			if _, err := stmt.Exec(i, pos/subtreeSize, encoded, encoded); err != nil {
+				return err
+			}
+		}
+		if elem.LeafIndex+1 > s.numLeaves {
+			s.numLeaves = elem.LeafIndex + 1
+		}
+		return nil
+	}
+
+	stmt, err := dbTxn.Prepare(`INSERT INTO merkle_proofs(i, j, hash) VALUES (?, ?, ?) ON CONFLICT (i, j) DO UPDATE SET hash = ?;`)
+	if err != nil {
+		return err
+	}
+
+	update.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) {
+		if err != nil {
+			return
+		}
+		err = modifyLeaf(stmt, sce.StateElement)
+		return
+	})
+	if err != nil {
+		return err
+	}
+
+	update.ForEachSiafundElement(func(sce types.SiafundElement, spent bool) {
+		if err != nil {
+			return
+		}
+		err = modifyLeaf(stmt, sce.StateElement)
+		return
+	})
+	if err != nil {
+		return err
+	}
+
+	update.ForEachFileContractElement(func(sce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) {
+		if err != nil {
+			return
+		}
+		err = modifyLeaf(stmt, sce.StateElement)
+		return
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MerkleProof implements explorer.Store.
+func (s *Store) MerkleProof(leafIndex uint64) ([]types.Hash256, error) {
+	proof := make([]types.Hash256, bits.Len64(leafIndex^s.numLeaves)-1)
+	err := s.transaction(func(tx txn) error {
+		pos := leafIndex
+		stmt, err := tx.Prepare("SELECT hash FROM merkle_proofs WHERE i = ? AND j = ?")
+		if err != nil {
+			return err
+		}
+		for i := range proof {
+			subtreeSize := uint64(1 << i)
+			if leafIndex&(1<<i) == 0 {
+				pos += subtreeSize
+			} else {
+				pos -= subtreeSize
+			}
+			// read hash (i, pos/subtreeSize)
+			if err := stmt.QueryRow(i, pos/subtreeSize).Scan(dbDecode(&proof[i])); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	return proof, err
+}

persist/sqlite/store.go

@@ -23,6 +23,7 @@ type (
 		log *zap.Logger
 
 		mu             sync.Mutex
+		numLeaves      uint64
 		pendingUpdates []*chain.ApplyUpdate
 	}
 )