update

internal/database/blockchainDB/buffered_file.go → internal/database/blockchainDB/bfile.go

@@ -79,6 +79,7 @@ type BFile struct {
 	EOB       int                    // End of the current buffer... Where to put the next data 'write'
 }
 
+
 // Get
 // Get the value for a given DBKeyFull
 func (b *BFile) Get(Key [32]byte) (value []byte, err error) {
@@ -159,7 +160,7 @@ func (b *BFile) CreateBuffers() {
 
 // NewBFile
 // Creates a new Buffered file.  The caller is responsible for writing the header
-func NewBFile(BufferCnt int, Filename string) (bFile *BFile, err error) {
+func NewBFile( Filename string, BufferCnt int) (bFile *BFile, err error) {
 	bFile = new(BFile)                     // create a new BFile
 	bFile.Keys = make(map[[32]byte]DBBKey) // Allocate the Keys map
 	bFile.BufferCnt = BufferCnt            // How many buffers we are going to use
@@ -221,6 +222,8 @@ func (b *BFile) Write(Data []byte) error {
 	return b.Write(Data)              // Write out the remaining data
 }
 
+
+
 // OpenBFile
 // Open a DBBlock file at a given height for read/write access
 // The only legitimate writes to a BFile would be to add/update keys

internal/database/blockchainDB/buffered_file_test.go → internal/database/blockchainDB/bfile_test.go

@@ -30,7 +30,7 @@ var KeySliceDir = filepath.Join(Directory, "keySlice")
 func TestWriteSmallKeys(t *testing.T) {
 
 	filename := filepath.Join(os.TempDir(), "BFileTest.dat")
-	bFile, err := NewBFile(5, filename)
+	bFile, err := NewBFile( filename,5)
 	assert.NoError(t, err, "expected no error creating BBFile")
 
 	getKey := func(v byte) (r [32]byte) {
@@ -56,7 +56,7 @@ func TestWriteKeys(t *testing.T) {
 
 	start := time.Now()
 	filename := filepath.Join(os.TempDir(), "BFileTest.dat")
-	bFile, err := NewBFile(5, filename)
+	bFile, err := NewBFile(filename,5)
 	assert.NoError(t, err, "expected no error creating BBFile")
 
 	fr := NewFastRandom([32]byte{}) // Make a random number generator
@@ -79,7 +79,7 @@ func TestReadKeys(t *testing.T) {
 	os.Mkdir(Directory, os.ModePerm)
 
 	filename := filepath.Join(os.TempDir(), "BFileTest.dat")
-	bFile, err := NewBFile(5, filename)
+	bFile, err := NewBFile(filename,5)
 	assert.NoError(t, err, "expected no error creating BBFile")
 
 	fr := NewFastRandom([32]byte{}) // Make a random number generator

internal/database/blockchainDB/bfile_writer.go → internal/database/blockchainDB/bfilewriter.go

@@ -48,10 +48,6 @@ func (b *BFileWriter) process() {
 			if _, err := b.File.Write(buff[:]); err != nil {   //
 				panic(err)
 			}
-
-			if _, err := b.File.Seek(0, io.SeekEnd); err != nil { // Seek to end
-				panic(err)
-			}
 			b.File.Close()
 			b.BuffPool <- c.buffer
 			return

internal/database/blockchainDB/block_list.go → internal/database/blockchainDB/blocklist.go

@@ -99,7 +99,7 @@ func (b *BlockList) NextBlockFile() (err error) {
 	b.BlockHeight++
 
 	filename := b.GetFilename(b.BlockHeight)
-	if b.BFile, err = NewBFile(b.BufferCnt, filename); err != nil {
+	if b.BFile, err = NewBFile( filename,b.BufferCnt); err != nil {
 		return err
 	}
 
@@ -113,7 +113,7 @@ func NewBlockFile(Directory string, BufferCnt int) (blockFile *BlockList, err er
 	blockFile.LoadState()
 
 	filename := blockFile.GetFilename(blockFile.BlockHeight)
-	if blockFile.BFile, err = NewBFile(BufferCnt, filename); err != nil {
+	if blockFile.BFile, err = NewBFile(filename,BufferCnt); err != nil {
 		return nil, err
 	}
 

internal/database/blockchainDB/shard.go

@@ -0,0 +1,71 @@
+package blockchainDB
+
+import (
+	"os"
+	"sync"
+)
+
+// Shard
+// Holds the stuff required to access a shard.
+type Shard struct {
+	BufferCnt int                 // Number of buffers to use
+	Filename  string              // The file with the BFile
+	BFile     *BFile              // The BFile
+	Cache     map[[32]byte][]byte // Cache of what is being written to the cache
+	Mutex     sync.Mutex          // Keeps compression from conflict with access
+}
+
+// NewShard
+// Create and open a new Shard
+func NewShard(BufferCnt int, Filename string) (shard *Shard, err error) {
+	shard = new(Shard)
+	shard.BufferCnt = BufferCnt
+	shard.Filename = Filename
+	shard.Cache = make(map[[32]byte][]byte)
+	if shard.BFile, err = NewBFile(Filename, BufferCnt); err != nil {
+		return nil, err
+	}
+	return shard, err
+}
+
+// Close
+// Clean up and close the Shard
+func (s *Shard) Close() {
+	s.Mutex.Lock()
+	defer s.Mutex.Unlock()
+
+	s.Cache = nil
+	s.BFile.Close()
+	s.BFile = nil
+}
+
+// Open
+// Open an existing Shard.  If the BFile does not exist, create it
+func (s *Shard) Open() (err error) {
+	if s.BFile != nil {
+		return
+	}
+	if s.BFile, err = OpenBFile(s.BufferCnt, s.Filename); err != nil {
+		if os.IsNotExist(err) {
+			s.BFile, err = NewBFile(s.Filename, s.BufferCnt)
+		}
+		return err
+	}
+	return nil
+}
+
+// Put
+// Put a key value pair into the shard
+func (s *Shard) Put (key [32]byte, value []byte) error {
+	s.Cache[key]=value
+	return s.BFile.Put(key,value)
+}
+
+// Get
+// Get the value for the key out of the shard
+func (s *Shard) Get (key [32]byte) ( value []byte, err error) {
+	if value, ok := s.Cache[key]; ok {
+		return value, nil
+	}
+	return s.BFile.Get(key)
+}

internal/database/blockchainDB/shard_db.go → internal/database/blockchainDB/sharddb.go

@@ -2,37 +2,63 @@ package blockchainDB
 
 import (
 	"encoding/binary"
-	"sync"
+	"fmt"
+	"os"
+	"path/filepath"
 )
 
 const (
 	ShardBits = 9
 	Shards    = 512 // Number of shards in bits
 )
 
-// Shard
-// Holds the stuff required to access a shard.
-type Shard struct {
-	File  string     // The file with the BFile
-	BFile *BFile     // The BFile
-	Mutex sync.Mutex // Keeps compression from conflict with access
-}
-
 // ShardDB
 // Maintains shards of key value pairs to allow reading and writing of
 // key value pairs even during compression and eventually multi-thread
 // transactions.
 type ShardDB struct {
-	PermBFile *BFile         // The BFile has the directory and file
+	PermBFile *BlockList     // The BFile has the directory and file
+	BufferCnt int            // Buffer count used for BFiles
 	Shards    [Shards]*Shard // List of all the Shards
 }
 
-func (s *ShardDB) Create(Directory string) (err error) {
-	//	if s.PermBFile, err = NewBFile(5, Directory, BFilePerm, BFileDN); err != nil {
-	//		return err
-	//	}
+func CreateShardDB(Directory string, Partition, BufferCnt int) (SDB *ShardDB, err error) {
+	_, err = os.Stat(Directory)
+	if err == nil {
+		return nil, fmt.Errorf("cannot create ShardDB; directory %s exists", Directory)
+	}
+	if !os.IsNotExist(err) {
+		return nil, fmt.Errorf("error getting status on directory '%s': %v", Directory, err)
+	}
+	SDB = new(ShardDB)
+	SDB.BufferCnt = BufferCnt
+	err = os.Mkdir(Directory, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+	SDB.PermBFile, err = NewBlockList(filepath.Join(Directory, "PermBFile"), Partition, BufferCnt)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < Shards; i++ {
+		sDir := filepath.Join(Directory, fmt.Sprintf("shard%03d-%03d", Partition, i))
+		err = os.Mkdir(sDir, os.ModePerm)
+		if err != nil {
+			os.RemoveAll(Directory)
+			return nil, err
+		}
+		SDB.Shards[i] = new(Shard)
+		SDB.Shards[i].Filename = filepath.Join(sDir, "shard.dat")
+		SDB.Shards[i].BufferCnt = BufferCnt
+		err = SDB.Shards[i].Open()
+		if err != nil {
+			os.RemoveAll(Directory)
+			return nil, err
+		}
+	}
 
-	return nil
+	return SDB, nil
 }
 
 func (s *ShardDB) Close() {
@@ -46,25 +72,6 @@ func (s *ShardDB) Close() {
 	}
 }
 
-/*
-func (s *ShardDB) Open(Directory string) (err error) {
-
-	if s.PermBFile, err = OpenBFileList(Directory, BFilePerm, BFileDN, 0); err != nil {
-		return err
-	}
-
-	for i := 0; i < Shards; i++ {
-		s.Shards[i] = new(Shard)
-		s.Shards[i].filename= filepath.Join(Directory, fmt.Sprintf("shard-%03x", i))
-		s.Shards[i].BFile, err = OpenBFileList(SDir, BFilePerm, BFileDN, 0)
-
-	}
-}
-*/
-// PutH
-// When the key is the hash (or other function) of the value, where the value will
-// never change, then use PutH.  The assumption is that these values, once recorded,
-// will not be used in a validator.
 func (s *ShardDB) PutH(scratch bool, key [32]byte, value []byte) error {
 	k := binary.BigEndian.Uint16(key[:]) >> (16 - ShardBits)
 	shard := s.Shards[k]
@@ -82,7 +89,7 @@ func (s *ShardDB) Put(key [32]byte, value []byte) error {
 	shard := s.Shards[k]
 	if shard == nil {
 		shard = new(Shard)
-		//shard.Open()
+		shard.Open()
 		s.Shards[k] = shard
 	}
 	return shard.BFile.Put(key, value)

internal/database/blockchainDB/sharddb_test.go

@@ -0,0 +1,55 @@
+package blockchainDB
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gitlab.com/accumulatenetwork/accumulate/internal/database/smt/common"
+)
+
+func TestShardDB(t *testing.T) {
+	var shardDB ShardDB
+	var r common.RandHash
+	for i := 0; i < 1000000; i++ {
+		key := r.NextA()
+		value := r.GetRandBuff(200)
+		shardDB.Put(key, value)
+	}
+	r = *new(common.RandHash)
+	for i := 0; i < 1000000; i++ {
+		key := r.NextA()
+		value := r.GetRandBuff(200)
+		v := shardDB.Get(key)
+		assert.Equal(t, value, v, "did not get the same value back")
+	}
+}
+
+func TestShard(t *testing.T) {
+	directory := filepath.Join(os.TempDir(),"ShardTest")
+	os.Mkdir(directory,os.ModePerm)
+	defer os.RemoveAll(directory)
+	filename := filepath.Join(directory,"shard")
+
+	shard,err := NewShard(5,filename)
+	assert.NoError(t,err,err)
+
+	entries := make(map[[32]byte][]byte)
+	fr := NewFastRandom([32]byte {1,2,3,4,5})
+	for i:= 0; i<100000; i++ {
+		entries[fr.NextHash()] = fr.RandBuff(100,500)
+	}
+	for k := range entries {
+		nv := fr.RandBuff(100,500)
+		shard.Put(k,nv)
+		entries[k]=nv
+	}
+	for i:=0;i<1000000,i++ {
+		v2,err := shard.Get(k)
+		assert.NoError(t,err,err)
+		assert.Equal(t,v,v2,"Didn't get the right value back")
+	}
+
+	shard.Close()
+}