use radixtree instead of art

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+*.pprof
 .vscode
 _bench*
 # Test binary, built with `go test -c`

db.go

@@ -6,9 +6,10 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 
-	art "github.com/WenyXu/sync-adaptive-radix-tree"
+	"github.com/millken/archivedb/internal/radixtree"
 	"github.com/pkg/errors"
 )
 
@@ -35,19 +36,19 @@ var (
 type DB struct {
 	path     string
 	opts     *option
-	index    art.Tree[*index]
+	index    *radixtree.Tree[*index]
 	segments []*segment
 	mu       sync.RWMutex
 }
 
 func Open(path string, options ...Option) (db *DB, err error) {
 	opts := &option{
-		fsync:    false,
-		hashFunc: DefaultHashFunc,
+		fsync: false,
 	}
 	db = &DB{
-		path: path,
-		opts: opts,
+		path:  path,
+		opts:  opts,
+		index: radixtree.New[*index](),
 	}
 	// Create path if it doesn't exist.
 	if err := os.MkdirAll(filepath.Join(path), 0777); err != nil {
@@ -65,9 +66,9 @@ func Open(path string, options ...Option) (db *DB, err error) {
 			return err
 		}
 
-		//  if err := db.index.Recover(db.segments); err != nil {
-		// 	return err
-		// }
+		if err := loadIndexes(db.index, db.segments); err != nil {
+			return err
+		}
 
 		return nil
 	}(); err != nil {
@@ -94,6 +95,9 @@ func (db *DB) openSegments() error {
 			return err
 		}
 		db.segments = append(db.segments, segment)
+		sort.Slice(db.segments, func(i, j int) bool {
+			return db.segments[i].id < db.segments[j].id
+		})
 	}
 	// Create initial segment if none exist.
 	if len(db.segments) == 0 {
@@ -163,7 +167,7 @@ func (db *DB) set(key, value []byte, flag flag) error {
 		return err
 	}
 	offset := segment.Size() - entry.Size()
-	db.index.Insert(key, &index{
+	db.index.Put(key, &index{
 		seg: segment.id,
 		off: offset,
 	})
@@ -182,7 +186,7 @@ func (db *DB) Get(key []byte) ([]byte, error) {
 	if err := validateKey(key); err != nil {
 		return nil, err
 	}
-	idx, found := db.index.Search(key)
+	idx, found := db.index.Get(key)
 	if !found {
 		return nil, ErrKeyNotFound
 	}

db_test.go

@@ -3,8 +3,6 @@ package archivedb
 import (
 	"bytes"
 	"fmt"
-	"hash/adler32"
-	"hash/crc32"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -24,49 +22,33 @@ type benchmarkTestCase struct {
 	size int
 }
 
-func BenchmarkHashFunc(b *testing.B) {
-	crc32Hash := func(data []byte) uint64 {
-		return uint64(crc32.ChecksumIEEE(data))
-	}
-	adler32Hash := func(data []byte) uint64 {
-		return uint64(adler32.Checksum(data))
-	}
-	testSize := []struct {
-		name string
-		size int
-	}{
-		{"128B", 128},
-		{"256B", 256},
-		{"1K", 1024},
-		{"2K", 2048},
-		{"4K", 4096},
-		{"8K", 8192},
-		{"16K", 16384},
-		{"32K", 32768},
-		{"64K", 65536},
-		{"128K", 131072},
-	}
-	tests := []struct {
-		hash HashFunc
-		name string
-	}{
-		{DefaultHashFunc, "DefaultHashFunc"},
-		{adler32Hash, "adler32"},
-		{crc32Hash, "crc32"},
+func TestDB_Reopen(t *testing.T) {
+	require := require.New(t)
+	dir, cleanup := MustTempDir()
+	defer cleanup()
+	db, err := Open(dir)
+	require.NoError(err)
+	require.NotNil(db)
+
+	numKeys := 1000000
+	for i := 0; i < numKeys; i++ {
+		key := []byte(fmt.Sprintf("%016d", i))
+		value := []byte(fmt.Sprintf("%d", i))
+		err = db.Put(key, value)
+		require.NoError(err)
 	}
-	for _, size := range testSize {
-		b.Run(size.name, func(b *testing.B) {
-			val := bytes.Repeat([]byte{' '}, size.size)
-			for _, test := range tests {
-				b.Run(test.name, func(b *testing.B) {
-					for i := 0; i < b.N; i++ {
-						test.hash(val)
-					}
-				})
-			}
-		})
+	require.NoError(db.Close())
+	db, err = Open(dir)
+	require.NoError(err)
+	require.NotNil(db)
+	for i := 0; i < numKeys; i++ {
+		key := []byte(fmt.Sprintf("%016d", i))
+		value, err := db.Get(key)
+		require.NoError(err, "key: %s", key)
+		require.Equal([]byte(fmt.Sprintf("%d", i)), value)
 	}
 }
+
 func TestDB(t *testing.T) {
 	require := require.New(t)
 	dir, cleanup := MustTempDir()

go.mod

@@ -3,8 +3,6 @@ module github.com/millken/archivedb
 go 1.20
 
 require (
-	github.com/WenyXu/sync-adaptive-radix-tree v0.0.0-20221020123713-1ae3c4a8dd92
-	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/sys v0.0.0-20221010170243-090e33056c14

go.sum

@@ -1,19 +1,6 @@
-github.com/WenyXu/sync-adaptive-radix-tree v0.0.0-20221020123713-1ae3c4a8dd92 h1:G28oV60J2+xbzC5M3NSaPPHZMjJ1K/9v4wzCzl3adbc=
-github.com/WenyXu/sync-adaptive-radix-tree v0.0.0-20221020123713-1ae3c4a8dd92/go.mod h1:DssQPBoTAUboJ5fFinarazaksPe8p/u4wG19wnzcxrk=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger/v3 v3.2103.3 h1:s63J1pisDhKpzWslXFe+ChuthuZptpwTE6qEKoczPb4=
-github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
-github.com/dshulyak/art v0.0.0-20200731100216-8869b840fedc h1:LW4aUpRQdTqQbell1egWpq4YjlHq+cdV/hZjRyihip4=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
-github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -23,8 +10,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/tidwall/btree v1.4.4 h1:tOsRz2Upq6BEJz8T++C6THzNh9xGWymBOOSfA7ffNXY=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

index.go

@@ -1,6 +1,41 @@
 package archivedb
 
+import "github.com/millken/archivedb/internal/radixtree"
+
 type index struct {
 	seg uint16
 	off uint32
 }
+
+func loadIndexes(idx *radixtree.Tree[*index], segments []*segment) error {
+	for _, segment := range segments {
+		for size := uint32(SegmentHeaderSize); size < segment.Size(); {
+			buf, err := segment.mmap.ReadOff(int(size), hdrSize)
+			if err != nil {
+				return err
+			}
+			h := hdr(buf)
+			if !h.getFlag().isEntryValid() {
+				break
+			}
+			off := int(size) + hdrSize
+			key, err := segment.mmap.ReadOff(off, int(h.getKeySize()))
+			if err != nil {
+				return err
+			}
+
+			switch h.getFlag() {
+			case flagEntryPut:
+				idx.Put(key, &index{
+					seg: segment.id,
+					off: uint32(size),
+				})
+			case flagEntryDelete:
+				idx.Delete(key)
+			}
+			size += h.entrySize()
+
+		}
+	}
+	return nil
+}