From 18cc3efda9120a167eeaa3413d029a6b3c592217 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 10 Aug 2022 10:09:37 +0100 Subject: [PATCH 1/4] refactored options passed when opening db --- embedded/bolted.go | 21 ++++++----- embedded/bolted_test.go | 36 +++++++++---------- embedded/observer_test.go | 4 +-- embedded/options.go | 31 ---------------- features/bolted_features_test.go | 2 +- metrics/metrics_write_tx_decorator_test.go | 10 +++--- replicated/replica/replica.go | 4 +-- replicated/txstream/features/txstream_test.go | 4 +-- 8 files changed, 40 insertions(+), 72 deletions(-) delete mode 100644 embedded/options.go diff --git a/embedded/bolted.go b/embedded/bolted.go index 22bdd32..500776e 100644 --- a/embedded/bolted.go +++ b/embedded/bolted.go @@ -9,19 +9,23 @@ import ( "github.com/draganm/bolted/dbpath" "go.etcd.io/bbolt" - bolt "go.etcd.io/bbolt" ) type Bolted struct { - db *bolt.DB + db *bbolt.DB obs *observer writeTxDecorators []WriteTxDecorator } +type Options struct { + bbolt.Options + WriteDecorators []WriteTxDecorator +} + const rootBucketName = "root" -func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) { - db, err := bolt.Open(path, mode, &bolt.Options{}) +func Open(path string, mode os.FileMode, options Options) (*Bolted, error) { + db, err := bbolt.Open(path, mode, &options.Options) if err != nil { return nil, fmt.Errorf("while opening bolt db: %w", err) } @@ -40,7 +44,7 @@ func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) { } if !rootExists { - err = db.Update(func(tx *bolt.Tx) error { + err = db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(rootBucketName)) if b == nil { _, err := tx.CreateBucket([]byte(rootBucketName)) @@ -65,12 +69,7 @@ func Open(path string, mode os.FileMode, options ...Option) (*Bolted, error) { writeTxDecorators: []WriteTxDecorator{obs.writeTxDecorator}, } - for _, o := range options { - err = o(b) - if err != nil { - return nil, fmt.Errorf("while applying option: %w", err) - } - } + b.writeTxDecorators = append(b.writeTxDecorators, options.WriteDecorators...) return b, nil diff --git a/embedded/bolted_test.go b/embedded/bolted_test.go index 6c90338..6423835 100644 --- a/embedded/bolted_test.go +++ b/embedded/bolted_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func openEmptyDatabase(t *testing.T, opts ...embedded.Option) (bolted.Database, func()) { +func openEmptyDatabase(t *testing.T, opts embedded.Options) (bolted.Database, func()) { td, err := ioutil.TempDir("", "") require.NoError(t, err) removeTempDir := func() { @@ -20,7 +20,7 @@ func openEmptyDatabase(t *testing.T, opts ...embedded.Option) (bolted.Database, require.NoError(t, err) } - db, err := embedded.Open(filepath.Join(td, "db"), 0660, opts...) + db, err := embedded.Open(filepath.Join(td, "db"), 0660, opts) require.NoError(t, err) @@ -37,14 +37,14 @@ func openEmptyDatabase(t *testing.T, opts ...embedded.Option) (bolted.Database, } func TestOpen(t *testing.T) { - _, cleanup := openEmptyDatabase(t) + _, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() } func TestCreateMap(t *testing.T) { t.Run("create map", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { tx.CreateMap(dbpath.ToPath("test")) @@ -54,7 +54,7 @@ func TestCreateMap(t *testing.T) { }) t.Run("create map twice", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -72,7 +72,7 @@ func TestCreateMap(t *testing.T) { }) t.Run("create map nested", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -105,7 +105,7 @@ func TestCreateMap(t *testing.T) { func TestDelete(t *testing.T) { t.Run("delete not existing map", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -116,7 +116,7 @@ func TestDelete(t *testing.T) { }) t.Run("delete existing map", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -133,7 +133,7 @@ func TestDelete(t *testing.T) { }) t.Run("delete parent map", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -151,7 +151,7 @@ func TestDelete(t *testing.T) { }) t.Run("delete child map", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -169,7 +169,7 @@ func TestDelete(t *testing.T) { }) t.Run("delete value", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -190,7 +190,7 @@ func TestDelete(t *testing.T) { func TestPutAndGet(t *testing.T) { t.Run("put and get to root", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -213,7 +213,7 @@ func TestPutAndGet(t *testing.T) { }) t.Run("put and get to map root", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -265,7 +265,7 @@ func TestPutAndGet(t *testing.T) { func TestIterator(t *testing.T) { t.Run("iterating empty root", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -278,7 +278,7 @@ func TestIterator(t *testing.T) { }) t.Run("iterating root with one value", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -304,7 +304,7 @@ func TestIterator(t *testing.T) { }) t.Run("iterating root with two values", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -350,7 +350,7 @@ func TestIterator(t *testing.T) { }) t.Run("iterating root with two values and a bucket", func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() err := bolted.SugaredWrite(db, func(tx bolted.SugaredWriteTx) error { @@ -463,7 +463,7 @@ func TestSize(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { - db, cleanup := openEmptyDatabase(t) + db, cleanup := openEmptyDatabase(t, embedded.Options{}) defer cleanup() var sz uint64 diff --git a/embedded/observer_test.go b/embedded/observer_test.go index 2b697a8..16ce83a 100644 --- a/embedded/observer_test.go +++ b/embedded/observer_test.go @@ -6,13 +6,13 @@ import ( "github.com/draganm/bolted" "github.com/draganm/bolted/dbpath" + "github.com/draganm/bolted/embedded" "github.com/stretchr/testify/require" ) func TestObservePath(t *testing.T) { - db, cleanupDatabase := openEmptyDatabase(t) - + db, cleanupDatabase := openEmptyDatabase(t, embedded.Options{}) defer cleanupDatabase() updates, close := db.Observe(dbpath.ToPath("foo").ToMatcher().AppendAnySubpathMatcher()) diff --git a/embedded/options.go b/embedded/options.go deleted file mode 100644 index 3493ad4..0000000 --- a/embedded/options.go +++ /dev/null @@ -1,31 +0,0 @@ -package embedded - -import "errors" - -type Option func(*Bolted) error - -func WithWriteTxDecorators(d ...WriteTxDecorator) func(*Bolted) error { - return func(b *Bolted) error { - for _, cl := range d { - if cl == nil { - return errors.New("writeTx decorator must not be nil") - } - } - b.writeTxDecorators = append(b.writeTxDecorators, d...) - return nil - } -} - -func WithNoSync() func(*Bolted) error { - return func(b *Bolted) error { - b.db.NoSync = true - return nil - } -} - -func WithNoGrowSync() func(*Bolted) error { - return func(b *Bolted) error { - b.db.NoGrowSync = true - return nil - } -} diff --git a/features/bolted_features_test.go b/features/bolted_features_test.go index 1697338..4b80a7d 100644 --- a/features/bolted_features_test.go +++ b/features/bolted_features_test.go @@ -30,7 +30,7 @@ var _ = steps.Then("the database is open", func(w *world.World) error { return os.RemoveAll(td) }) - db, err := embedded.Open(filepath.Join(td, "db"), 0700) + db, err := embedded.Open(filepath.Join(td, "db"), 0700, embedded.Options{}) if err != nil { return err } diff --git a/metrics/metrics_write_tx_decorator_test.go b/metrics/metrics_write_tx_decorator_test.go index 31415c8..41fae2b 100644 --- a/metrics/metrics_write_tx_decorator_test.go +++ b/metrics/metrics_write_tx_decorator_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -func openEmptyDatabase(t *testing.T, opts ...embedded.Option) (bolted.Database, func()) { +func openEmptyDatabase(t *testing.T, opts embedded.Options) (bolted.Database, func()) { td, err := ioutil.TempDir("", "") require.NoError(t, err) removeTempDir := func() { @@ -23,7 +23,7 @@ func openEmptyDatabase(t *testing.T, opts ...embedded.Option) (bolted.Database, require.NoError(t, err) } - db, err := embedded.Open(filepath.Join(td, "db"), 0660, opts...) + db, err := embedded.Open(filepath.Join(td, "db"), 0660, opts) require.NoError(t, err) @@ -70,7 +70,7 @@ func findMetricWithName(t *testing.T, name string) *dto.Metric { func TestMetrics(t *testing.T) { t.Run("number of write transactions", func(t *testing.T) { - db, cleanupDatabase := openEmptyDatabase(t, embedded.WithWriteTxDecorators(metrics.NewWriteTxDecorator(t.Name()))) + db, cleanupDatabase := openEmptyDatabase(t, embedded.Options{WriteDecorators: []embedded.WriteTxDecorator{metrics.NewWriteTxDecorator(t.Name())}}) defer cleanupDatabase() @@ -91,7 +91,7 @@ func TestMetrics(t *testing.T) { t.Run("number of successful transactions", func(t *testing.T) { - db, cleanupDatabase := openEmptyDatabase(t, embedded.WithWriteTxDecorators(metrics.NewWriteTxDecorator(t.Name()))) + db, cleanupDatabase := openEmptyDatabase(t, embedded.Options{WriteDecorators: []embedded.WriteTxDecorator{metrics.NewWriteTxDecorator(t.Name())}}) defer cleanupDatabase() @@ -111,7 +111,7 @@ func TestMetrics(t *testing.T) { }) t.Run("number of failed transactions", func(t *testing.T) { - db, cleanupDatabase := openEmptyDatabase(t, embedded.WithWriteTxDecorators(metrics.NewWriteTxDecorator(t.Name()))) + db, cleanupDatabase := openEmptyDatabase(t, embedded.Options{WriteDecorators: []embedded.WriteTxDecorator{metrics.NewWriteTxDecorator(t.Name())}}) defer cleanupDatabase() diff --git a/replicated/replica/replica.go b/replicated/replica/replica.go index 04c1ca9..22d6bb5 100644 --- a/replicated/replica/replica.go +++ b/replicated/replica/replica.go @@ -63,7 +63,7 @@ func Open(ctx context.Context, primaryURL, dbPath string) (Replica, error) { cond: cond, } - embedded, err := embedded.Open(dbPath, 0700, embedded.WithWriteTxDecorators(func(tx bolted.WriteTx) bolted.WriteTx { + embedded, err := embedded.Open(dbPath, 0700, embedded.Options{WriteDecorators: []embedded.WriteTxDecorator{func(tx bolted.WriteTx) bolted.WriteTx { return &writeTxNumberListener{tx, func(lastTXID uint64) { r.mu.Lock() r.lastTXID = lastTXID @@ -71,7 +71,7 @@ func Open(ctx context.Context, primaryURL, dbPath string) (Replica, error) { r.mu.Unlock() }} - })) + }}}) if err != nil { cancel() return nil, fmt.Errorf("while opening local embedded db: %w", err) diff --git a/replicated/txstream/features/txstream_test.go b/replicated/txstream/features/txstream_test.go index 9190618..4bdc5a2 100644 --- a/replicated/txstream/features/txstream_test.go +++ b/replicated/txstream/features/txstream_test.go @@ -33,7 +33,7 @@ var _ = steps.Then("empty source and destination database", func(w *world.World) return os.RemoveAll(td) }) - source, err := embedded.Open(filepath.Join(td, "source"), 0700) + source, err := embedded.Open(filepath.Join(td, "source"), 0700, embedded.Options{}) if err != nil { return fmt.Errorf("while opening source db: %w", err) } @@ -43,7 +43,7 @@ var _ = steps.Then("empty source and destination database", func(w *world.World) w.AddCleanup(source.Close) - destination, err := embedded.Open(filepath.Join(td, "destination"), 0700) + destination, err := embedded.Open(filepath.Join(td, "destination"), 0700, embedded.Options{}) if err != nil { return fmt.Errorf("while opening destination db: %w", err) } From 290b74e373334717e6cba6feec3a90bc6308c18c Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 10 Aug 2022 11:07:25 +0100 Subject: [PATCH 2/4] added support for setting fill percent of buckets in transactions --- db.go | 1 + embedded/bolted.go | 7 ++++--- embedded/write_tx.go | 27 +++++++++++++++++++++++---- replicated/txstream/replay.go | 12 ++++++++++++ replicated/txstream/writer.go | 15 +++++++++++++++ sugared.go | 8 ++++++++ 6 files changed, 63 insertions(+), 7 deletions(-) diff --git a/db.go b/db.go index b8e6641..9e7efd4 100644 --- a/db.go +++ b/db.go @@ -19,6 +19,7 @@ type WriteTx interface { CreateMap(path dbpath.Path) error Delete(path dbpath.Path) error Put(path dbpath.Path, value []byte) error + SetFillPercent(float64) error Rollback() error ReadTx } diff --git a/embedded/bolted.go b/embedded/bolted.go index 500776e..5bd60f5 100644 --- a/embedded/bolted.go +++ b/embedded/bolted.go @@ -108,9 +108,10 @@ func (b *Bolted) BeginWrite() (bolted.WriteTx, error) { rootBucket := btx.Bucket([]byte(rootBucketName)) wtx := &writeTx{ - btx: btx, - readOnly: false, - rootBucket: rootBucket, + btx: btx, + readOnly: false, + rootBucket: rootBucket, + fillPercent: bbolt.DefaultFillPercent, } var realWriteTx bolted.WriteTx = wtx diff --git a/embedded/write_tx.go b/embedded/write_tx.go index cb394cc..a923e50 100644 --- a/embedded/write_tx.go +++ b/embedded/write_tx.go @@ -10,10 +10,11 @@ import ( ) type writeTx struct { - btx *bbolt.Tx - readOnly bool - rolledBack bool - rootBucket *bbolt.Bucket + btx *bbolt.Tx + readOnly bool + rolledBack bool + rootBucket *bbolt.Bucket + fillPercent float64 } func (w *writeTx) Finish() (err error) { @@ -48,6 +49,18 @@ func (w *writeTx) Rollback() (err error) { return nil } +func (w *writeTx) SetFillPercent(fillPercent float64) error { + if fillPercent < 0.1 { + return errors.New("fill percent is too low") + } + + if fillPercent > 1.0 { + return errors.New("fill percent is too high") + } + w.fillPercent = fillPercent + return nil +} + func (w *writeTx) CreateMap(path dbpath.Path) (err error) { defer func() { @@ -75,6 +88,8 @@ func (w *writeTx) CreateMap(path dbpath.Path) (err error) { last := path[len(path)-1] + bucket.FillPercent = w.fillPercent + _, err = bucket.CreateBucket([]byte(last)) if err != nil { @@ -126,6 +141,8 @@ func (w *writeTx) Delete(path dbpath.Path) (err error) { } }() + bucket.FillPercent = w.fillPercent + val := bucket.Get(last) if val != nil { err := bucket.Delete(last) @@ -186,6 +203,8 @@ func (w *writeTx) Put(path dbpath.Path, value []byte) (err error) { } }() + bucket.FillPercent = w.fillPercent + err = bucket.Put([]byte(last), value) if err == bbolt.ErrIncompatibleValue { diff --git a/replicated/txstream/replay.go b/replicated/txstream/replay.go index e9eaca4..8da8e3d 100644 --- a/replicated/txstream/replay.go +++ b/replicated/txstream/replay.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "math" "github.com/draganm/bolted" "github.com/draganm/bolted/dbpath" @@ -345,6 +346,17 @@ func Replay(r io.Reader, db bolted.Database) (txID uint64, err error) { return 0, err } + case SetFillPercent: + fpint, err := binary.ReadUvarint(br) + if err != nil { + return 0, fmt.Errorf("while reading fill percent: %w", err) + } + fillPercent := math.Float64frombits(fpint) + err = tx.SetFillPercent(fillPercent) + if err != nil { + return 0, err + } + default: return 0, errors.New("unsupported operation") diff --git a/replicated/txstream/writer.go b/replicated/txstream/writer.go index 7906dc8..1bca884 100644 --- a/replicated/txstream/writer.go +++ b/replicated/txstream/writer.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "math" "github.com/draganm/bolted" "github.com/draganm/bolted/dbpath" @@ -49,6 +50,10 @@ func WriteByte(b byte) func(w *bufio.Writer) error { } } +func WriteFloat64(f float64) func(w *bufio.Writer) error { + return writeVarUint64(math.Float64bits(f)) +} + func writeBool(v bool) func(w *bufio.Writer) error { return func(w *bufio.Writer) error { if v { @@ -471,3 +476,13 @@ func (i *iteratorWriter) Last() error { writeVarUint64(i.idx), ) } + +const SetFillPercent byte = 20 + +func (w *Writer) SetFillPercent(fillPercent float64) error { + return WriteAll( + w.log, + WriteByte(SetFillPercent), + WriteFloat64(fillPercent), + ) +} diff --git a/sugared.go b/sugared.go index d37a0fd..c37fe49 100644 --- a/sugared.go +++ b/sugared.go @@ -11,6 +11,7 @@ type SugaredWriteTx interface { CreateMap(path dbpath.Path) Delete(path dbpath.Path) Put(path dbpath.Path, value []byte) + SetFillPercent(float64) SugaredReadTx } @@ -52,6 +53,13 @@ func (rt sugaredWriteTx) GetRawWriteTX() WriteTx { return rt.tx } +func (st sugaredWriteTx) SetFillPercent(fillPercent float64) { + err := st.tx.SetFillPercent(fillPercent) + if err != nil { + panic(err) + } +} + func (st sugaredWriteTx) CreateMap(path dbpath.Path) { err := st.tx.CreateMap(path) if err != nil { From 8ab4a2259e66d3f0c2baeb0218bba509e671291f Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 10 Aug 2022 11:25:29 +0100 Subject: [PATCH 3/4] added compact cmd --- cmd/bolted/compact/commang.go | 82 +++++++++++++++++++++++++++++++++++ cmd/bolted/main.go | 24 ++++++++++ 2 files changed, 106 insertions(+) create mode 100644 cmd/bolted/compact/commang.go create mode 100644 cmd/bolted/main.go diff --git a/cmd/bolted/compact/commang.go b/cmd/bolted/compact/commang.go new file mode 100644 index 0000000..f46dae9 --- /dev/null +++ b/cmd/bolted/compact/commang.go @@ -0,0 +1,82 @@ +package compact + +import ( + "fmt" + "os" + "time" + + "github.com/urfave/cli/v2" + "go.etcd.io/bbolt" +) + +var Command = &cli.Command{ + Name: "compact", + Usage: "Create a compacted copy of the database", + ArgsUsage: " ", + Flags: []cli.Flag{ + &cli.IntFlag{ + Usage: "Page size for the new database. Defaults to OS page size", + Name: "page-size", + EnvVars: []string{"PAGE_SIZE"}, + Value: os.Getpagesize(), + }, + &cli.StringFlag{ + Usage: "Type of the freelist for the new database: array | map", + Name: "freelist-type", + EnvVars: []string{"FREELIST_TYPE"}, + Value: "array", + }, + &cli.DurationFlag{ + Usage: "timeout for opening source database", + Name: "open-timeout", + Value: 500 * time.Millisecond, + EnvVars: []string{"OPEN_TIMEOUT"}, + }, + }, + + Action: func(c *cli.Context) error { + if c.NArg() != 2 { + return fmt.Errorf("source and destination file names must be provided") + } + + sourceFile := c.Args().Get(0) + destinationFile := c.Args().Get(1) + + s, err := bbolt.Open(sourceFile, 0700, &bbolt.Options{Timeout: c.Duration("open-timeout"), ReadOnly: true}) + if err != nil { + return fmt.Errorf("while opening source db: %w", err) + } + + defer s.Close() + + flt, err := freelistType(c.String("freelist-type")) + if err != nil { + return err + } + + d, err := bbolt.Open(destinationFile, 0700, &bbolt.Options{Timeout: c.Duration("open-timeout"), NoSync: true, PageSize: c.Int("page-size"), FreelistType: flt}) + if err != nil { + return fmt.Errorf("while opening destination db: %w", err) + } + + err = bbolt.Compact(d, s, 100000000) + + if err != nil { + return fmt.Errorf("while compacting db: %w", err) + } + + return nil + + }, +} + +func freelistType(t string) (bbolt.FreelistType, error) { + switch t { + case "map": + return bbolt.FreelistMapType, nil + case "array": + return bbolt.FreelistArrayType, nil + default: + return "", fmt.Errorf("unknown freelist type: %s", t) + } +} diff --git a/cmd/bolted/main.go b/cmd/bolted/main.go new file mode 100644 index 0000000..2f27e1b --- /dev/null +++ b/cmd/bolted/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "log" + "os" + + "github.com/draganm/bolted/cmd/bolted/compact" + "github.com/urfave/cli/v2" +) + +func main() { + app := &cli.App{ + Name: "bolted", + Usage: "Command line utility to inspect and manipulate bolted database files", + HideVersion: true, + Commands: []*cli.Command{ + compact.Command, + }, + } + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} From c69b34d58e0e9afcc199bc6cc0a39dd6e89e86b4 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 10 Aug 2022 13:46:55 +0100 Subject: [PATCH 4/4] added ls and cat commands to the cli --- cmd/bolted/cat/command.go | 60 +++++++++++++++++++++++++++++++++++ cmd/bolted/ls/command.go | 67 +++++++++++++++++++++++++++++++++++++++ cmd/bolted/main.go | 4 +++ 3 files changed, 131 insertions(+) create mode 100644 cmd/bolted/cat/command.go create mode 100644 cmd/bolted/ls/command.go diff --git a/cmd/bolted/cat/command.go b/cmd/bolted/cat/command.go new file mode 100644 index 0000000..ba0d95d --- /dev/null +++ b/cmd/bolted/cat/command.go @@ -0,0 +1,60 @@ +package cat + +import ( + "fmt" + "os" + "time" + + "github.com/draganm/bolted" + "github.com/draganm/bolted/dbpath" + "github.com/draganm/bolted/embedded" + "github.com/urfave/cli/v2" + "go.etcd.io/bbolt" +) + +var Command = &cli.Command{ + Name: "cat", + Aliases: []string{"list"}, + Usage: "print value from the database path", + ArgsUsage: " ", + Flags: []cli.Flag{ + &cli.DurationFlag{ + Usage: "timeout for opening the database", + Name: "open-timeout", + Value: 500 * time.Millisecond, + EnvVars: []string{"OPEN_TIMEOUT"}, + }, + }, + Action: func(c *cli.Context) error { + if c.NArg() != 2 { + return fmt.Errorf("db file and path must be provided") + } + + sourceFile := c.Args().Get(0) + p := c.Args().Get(1) + dbp, err := dbpath.Parse(p) + if err != nil { + return fmt.Errorf("while parsing path %s: %w", p, err) + } + db, err := embedded.Open(sourceFile, 0700, embedded.Options{ + Options: bbolt.Options{ + Timeout: c.Duration("open-timeout"), + ReadOnly: true, + }, + }) + + if err != nil { + return fmt.Errorf("while opening database: %w", err) + } + + return bolted.SugaredRead(db, func(tx bolted.SugaredReadTx) error { + val := tx.Get(dbp) + _, err := os.Stdout.Write(val) + if err != nil { + return err + } + return nil + }) + + }, +} diff --git a/cmd/bolted/ls/command.go b/cmd/bolted/ls/command.go new file mode 100644 index 0000000..21dbe9a --- /dev/null +++ b/cmd/bolted/ls/command.go @@ -0,0 +1,67 @@ +package ls + +import ( + "fmt" + "time" + + "github.com/draganm/bolted" + "github.com/draganm/bolted/dbpath" + "github.com/draganm/bolted/embedded" + "github.com/urfave/cli/v2" + "go.etcd.io/bbolt" +) + +var Command = &cli.Command{ + Name: "ls", + Aliases: []string{"list"}, + Usage: "list one bucket in the database", + ArgsUsage: " [db path]", + Flags: []cli.Flag{ + &cli.DurationFlag{ + Usage: "timeout for opening the database", + Name: "open-timeout", + Value: 500 * time.Millisecond, + EnvVars: []string{"OPEN_TIMEOUT"}, + }, + }, + Action: func(c *cli.Context) error { + if c.NArg() < 1 { + return fmt.Errorf("db file and path must be provided") + } + + sourceFile := c.Args().Get(0) + + p := c.Args().Get(1) + if p == "" { + p = "/" + } + dbp, err := dbpath.Parse(p) + if err != nil { + return fmt.Errorf("while parsing path %s: %w", p, err) + } + + db, err := embedded.Open(sourceFile, 0700, embedded.Options{ + Options: bbolt.Options{ + Timeout: c.Duration("open-timeout"), + ReadOnly: true, + }, + }) + + if err != nil { + return fmt.Errorf("while opening database: %w", err) + } + + return bolted.SugaredRead(db, func(tx bolted.SugaredReadTx) error { + for it := tx.Iterator(dbp); !it.IsDone(); it.Next() { + + suffix := "" + if it.GetValue() == nil { + suffix = "/" + } + fmt.Println(it.GetKey() + suffix) + } + return nil + }) + + }, +} diff --git a/cmd/bolted/main.go b/cmd/bolted/main.go index 2f27e1b..94f5951 100644 --- a/cmd/bolted/main.go +++ b/cmd/bolted/main.go @@ -4,7 +4,9 @@ import ( "log" "os" + "github.com/draganm/bolted/cmd/bolted/cat" "github.com/draganm/bolted/cmd/bolted/compact" + "github.com/draganm/bolted/cmd/bolted/ls" "github.com/urfave/cli/v2" ) @@ -15,6 +17,8 @@ func main() { HideVersion: true, Commands: []*cli.Command{ compact.Command, + ls.Command, + cat.Command, }, } err := app.Run(os.Args)