Merge pull request #7 from draganm/tuning-options

refactored options passed when opening db

cmd/bolted/cat/command.go

@@ -0,0 +1,60 @@
+package cat
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/draganm/bolted"
+	"github.com/draganm/bolted/dbpath"
+	"github.com/draganm/bolted/embedded"
+	"github.com/urfave/cli/v2"
+	"go.etcd.io/bbolt"
+)
+
+var Command = &cli.Command{
+	Name:      "cat",
+	Aliases:   []string{"list"},
+	Usage:     "print value from the database path",
+	ArgsUsage: "<database file> <db path>",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Usage:   "timeout for opening the database",
+			Name:    "open-timeout",
+			Value:   500 * time.Millisecond,
+			EnvVars: []string{"OPEN_TIMEOUT"},
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if c.NArg() != 2 {
+			return fmt.Errorf("db file and path must be provided")
+		}
+
+		sourceFile := c.Args().Get(0)
+		p := c.Args().Get(1)
+		dbp, err := dbpath.Parse(p)
+		if err != nil {
+			return fmt.Errorf("while parsing path %s: %w", p, err)
+		}
+		db, err := embedded.Open(sourceFile, 0700, embedded.Options{
+			Options: bbolt.Options{
+				Timeout:  c.Duration("open-timeout"),
+				ReadOnly: true,
+			},
+		})
+
+		if err != nil {
+			return fmt.Errorf("while opening database: %w", err)
+		}
+
+		return bolted.SugaredRead(db, func(tx bolted.SugaredReadTx) error {
+			val := tx.Get(dbp)
+			_, err := os.Stdout.Write(val)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+	},
+}

cmd/bolted/compact/commang.go

@@ -0,0 +1,82 @@
+package compact
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/urfave/cli/v2"
+	"go.etcd.io/bbolt"
+)
+
+var Command = &cli.Command{
+	Name:      "compact",
+	Usage:     "Create a compacted copy of the database",
+	ArgsUsage: "<source db file> <destination db file>",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Usage:   "Page size for the new database. Defaults to OS page size",
+			Name:    "page-size",
+			EnvVars: []string{"PAGE_SIZE"},
+			Value:   os.Getpagesize(),
+		},
+		&cli.StringFlag{
+			Usage:   "Type of the freelist for the new database: array | map",
+			Name:    "freelist-type",
+			EnvVars: []string{"FREELIST_TYPE"},
+			Value:   "array",
+		},
+		&cli.DurationFlag{
+			Usage:   "timeout for opening source database",
+			Name:    "open-timeout",
+			Value:   500 * time.Millisecond,
+			EnvVars: []string{"OPEN_TIMEOUT"},
+		},
+	},
+
+	Action: func(c *cli.Context) error {
+		if c.NArg() != 2 {
+			return fmt.Errorf("source and destination file names must be provided")
+		}
+
+		sourceFile := c.Args().Get(0)
+		destinationFile := c.Args().Get(1)
+
+		s, err := bbolt.Open(sourceFile, 0700, &bbolt.Options{Timeout: c.Duration("open-timeout"), ReadOnly: true})
+		if err != nil {
+			return fmt.Errorf("while opening source db: %w", err)
+		}
+
+		defer s.Close()
+
+		flt, err := freelistType(c.String("freelist-type"))
+		if err != nil {
+			return err
+		}
+
+		d, err := bbolt.Open(destinationFile, 0700, &bbolt.Options{Timeout: c.Duration("open-timeout"), NoSync: true, PageSize: c.Int("page-size"), FreelistType: flt})
+		if err != nil {
+			return fmt.Errorf("while opening destination db: %w", err)
+		}
+
+		err = bbolt.Compact(d, s, 100000000)
+
+		if err != nil {
+			return fmt.Errorf("while compacting db: %w", err)
+		}
+
+		return nil
+
+	},
+}
+
+func freelistType(t string) (bbolt.FreelistType, error) {
+	switch t {
+	case "map":
+		return bbolt.FreelistMapType, nil
+	case "array":
+		return bbolt.FreelistArrayType, nil
+	default:
+		return "", fmt.Errorf("unknown freelist type: %s", t)
+	}
+}

cmd/bolted/ls/command.go

@@ -0,0 +1,67 @@
+package ls
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/draganm/bolted"
+	"github.com/draganm/bolted/dbpath"
+	"github.com/draganm/bolted/embedded"
+	"github.com/urfave/cli/v2"
+	"go.etcd.io/bbolt"
+)
+
+var Command = &cli.Command{
+	Name:      "ls",
+	Aliases:   []string{"list"},
+	Usage:     "list one bucket in the database",
+	ArgsUsage: "<database file> [db path]",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Usage:   "timeout for opening the database",
+			Name:    "open-timeout",
+			Value:   500 * time.Millisecond,
+			EnvVars: []string{"OPEN_TIMEOUT"},
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if c.NArg() < 1 {
+			return fmt.Errorf("db file and path must be provided")
+		}
+
+		sourceFile := c.Args().Get(0)
+
+		p := c.Args().Get(1)
+		if p == "" {
+			p = "/"
+		}
+		dbp, err := dbpath.Parse(p)
+		if err != nil {
+			return fmt.Errorf("while parsing path %s: %w", p, err)
+		}
+
+		db, err := embedded.Open(sourceFile, 0700, embedded.Options{
+			Options: bbolt.Options{
+				Timeout:  c.Duration("open-timeout"),
+				ReadOnly: true,
+			},
+		})
+
+		if err != nil {
+			return fmt.Errorf("while opening database: %w", err)
+		}
+
+		return bolted.SugaredRead(db, func(tx bolted.SugaredReadTx) error {
+			for it := tx.Iterator(dbp); !it.IsDone(); it.Next() {
+
+				suffix := ""
+				if it.GetValue() == nil {
+					suffix = "/"
+				}
+				fmt.Println(it.GetKey() + suffix)
+			}
+			return nil
+		})
+
+	},
+}

cmd/bolted/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/draganm/bolted/cmd/bolted/cat"
+	"github.com/draganm/bolted/cmd/bolted/compact"
+	"github.com/draganm/bolted/cmd/bolted/ls"
+	"github.com/urfave/cli/v2"
+)
+
+func main() {
+	app := &cli.App{
+		Name:        "bolted",
+		Usage:       "Command line utility to inspect and manipulate bolted database files",
+		HideVersion: true,
+		Commands: []*cli.Command{
+			compact.Command,
+			ls.Command,
+			cat.Command,
+		},
+	}
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

db.go

@@ -19,6 +19,7 @@ type WriteTx interface {
 	CreateMap(path dbpath.Path) error
 	Delete(path dbpath.Path) error
 	Put(path dbpath.Path, value []byte) error
+	SetFillPercent(float64) error
 	Rollback() error
 	ReadTx
 }

embedded/bolted.go

@@ -9,19 +9,23 @@ import (
 	"github.com/draganm/bolted/dbpath"
 
 	"go.etcd.io/bbolt"
-	bolt "go.etcd.io/bbolt"
 )
 
 type Bolted struct {
-	db                *bolt.DB
+	db                *bbolt.DB
 	obs               *observer
 	writeTxDecorators []WriteTxDecorator
 }
 
+type Options struct {
+	bbolt.Options
+	WriteDecorators []WriteTxDecorator
+}
+
 const rootBucketName = "root"
 
-func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) {
-	db, err := bolt.Open(path, mode, &bolt.Options{})
+func Open(path string, mode os.FileMode, options Options) (*Bolted, error) {
+	db, err := bbolt.Open(path, mode, &options.Options)
 	if err != nil {
 		return nil, fmt.Errorf("while opening bolt db: %w", err)
 	}
@@ -40,7 +44,7 @@ func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) {
 		}
 
 		if !rootExists {
-			err = db.Update(func(tx *bolt.Tx) error {
+			err = db.Update(func(tx *bbolt.Tx) error {
 				b := tx.Bucket([]byte(rootBucketName))
 				if b == nil {
 					_, err := tx.CreateBucket([]byte(rootBucketName))
@@ -65,12 +69,7 @@ func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) {
 		writeTxDecorators: []WriteTxDecorator{obs.writeTxDecorator},
 	}
 
-	for _, o := range options {
-		err = o(b)
-		if err != nil {
-			return nil, fmt.Errorf("while applying option: %w", err)
-		}
-	}
+	b.writeTxDecorators = append(b.writeTxDecorators, options.WriteDecorators...)
 
 	return b, nil
 
@@ -109,9 +108,10 @@ func (b *Bolted) BeginWrite() (bolted.WriteTx, error) {
 
 	rootBucket := btx.Bucket([]byte(rootBucketName))
 	wtx := &writeTx{
-		btx:        btx,
-		readOnly:   false,
-		rootBucket: rootBucket,
+		btx:         btx,
+		readOnly:    false,
+		rootBucket:  rootBucket,
+		fillPercent: bbolt.DefaultFillPercent,
 	}
 
 	var realWriteTx bolted.WriteTx = wtx