diff --git a/cmd/accumulated/run/config.yml b/cmd/accumulated/run/config.yml index 0f430b6db..d4569e9f5 100644 --- a/cmd/accumulated/run/config.yml +++ b/cmd/accumulated/run/config.yml @@ -177,6 +177,29 @@ MemoryStorage: fields: ~ BadgerStorage: + union: { type: storage } + non-binary: true + fields: + - name: Path + type: string + - name: Version + type: int + +BoltStorage: + union: { type: storage } + non-binary: true + fields: + - name: Path + type: string + +LevelDBStorage: + union: { type: storage } + non-binary: true + fields: + - name: Path + type: string + +ExpBlockDBStorage: union: { type: storage } non-binary: true fields: diff --git a/cmd/accumulated/run/enums.yml b/cmd/accumulated/run/enums.yml index a89745109..d78a92b3a 100644 --- a/cmd/accumulated/run/enums.yml +++ b/cmd/accumulated/run/enums.yml @@ -39,6 +39,13 @@ StorageType: value: 1 Badger: value: 2 + Bolt: + value: 3 + LevelDB: + value: 4 + + ExpBlockDB: + value: 1001 ConfigurationType: CoreValidator: diff --git a/cmd/accumulated/run/enums_gen.go b/cmd/accumulated/run/enums_gen.go index c1cd53916..01178a282 100644 --- a/cmd/accumulated/run/enums_gen.go +++ b/cmd/accumulated/run/enums_gen.go @@ -80,6 +80,15 @@ const StorageTypeMemory StorageType = 1 // StorageTypeBadger . const StorageTypeBadger StorageType = 2 +// StorageTypeBolt . +const StorageTypeBolt StorageType = 3 + +// StorageTypeLevelDB . +const StorageTypeLevelDB StorageType = 4 + +// StorageTypeExpBlockDB . +const StorageTypeExpBlockDB StorageType = 1001 + // GetEnumValue returns the value of the Configuration Type func (v ConfigurationType) GetEnumValue() uint64 { return uint64(v) } @@ -363,7 +372,7 @@ func (v StorageType) GetEnumValue() uint64 { return uint64(v) } func (v *StorageType) SetEnumValue(id uint64) bool { u := StorageType(id) switch u { - case StorageTypeMemory, StorageTypeBadger: + case StorageTypeMemory, StorageTypeBadger, StorageTypeBolt, StorageTypeLevelDB, StorageTypeExpBlockDB: *v = u return true } @@ -377,6 +386,12 @@ func (v StorageType) String() string { return "memory" case StorageTypeBadger: return "badger" + case StorageTypeBolt: + return "bolt" + case StorageTypeLevelDB: + return "levelDB" + case StorageTypeExpBlockDB: + return "expBlockDB" } return fmt.Sprintf("StorageType:%d", v) } @@ -388,6 +403,12 @@ func StorageTypeByName(name string) (StorageType, bool) { return StorageTypeMemory, true case "badger": return StorageTypeBadger, true + case "bolt": + return StorageTypeBolt, true + case "leveldb": + return StorageTypeLevelDB, true + case "expblockdb": + return StorageTypeExpBlockDB, true } return 0, false } diff --git a/cmd/accumulated/run/storage.go b/cmd/accumulated/run/storage.go index 8c5a08d09..710012bf1 100644 --- a/cmd/accumulated/run/storage.go +++ b/cmd/accumulated/run/storage.go @@ -7,9 +7,14 @@ package run import ( + "io" + "gitlab.com/accumulatenetwork/accumulate/exp/ioc" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/badger" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/block" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/bolt" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/leveldb" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory" "golang.org/x/exp/slog" ) @@ -84,17 +89,64 @@ func (s *MemoryStorage) open(inst *Instance) (keyvalue.Beginner, error) { } func (s *BadgerStorage) open(inst *Instance) (keyvalue.Beginner, error) { - db, err := badger.New(inst.path(s.Path)) + var db interface { + keyvalue.Beginner + io.Closer + } + var err error + switch s.Version { + case 0, 1: + db, err = badger.OpenV1(inst.path(s.Path)) + case 2: + db, err = badger.OpenV2(inst.path(s.Path)) + case 3: + db, err = badger.OpenV3(inst.path(s.Path)) + case 4: + db, err = badger.OpenV4(inst.path(s.Path)) + } if err != nil { return nil, err } - inst.cleanup(func() { - err := db.Close() + inst.cleanupCloser(db, "Error while closing database") + return db, nil +} + +func (s *BoltStorage) open(inst *Instance) (keyvalue.Beginner, error) { + db, err := bolt.Open(inst.path(s.Path)) + if err != nil { + return nil, err + } + + inst.cleanupCloser(db, "Error while closing database") + return db, nil +} + +func (s *LevelDBStorage) open(inst *Instance) (keyvalue.Beginner, error) { + db, err := leveldb.Open(inst.path(s.Path)) + if err != nil { + return nil, err + } + + inst.cleanupCloser(db, "Error while closing database") + return db, nil +} + +func (s *ExpBlockDBStorage) open(inst *Instance) (keyvalue.Beginner, error) { + db, err := block.Open(inst.path(s.Path)) + if err != nil { + return nil, err + } + + inst.cleanupCloser(db, "Error while closing database") + return db, nil +} + +func (i *Instance) cleanupCloser(c io.Closer, msg string) { + i.cleanup(func() { + err := c.Close() if err != nil { - slog.ErrorCtx(inst.context, "Error while closing Badger", "error", err) + slog.ErrorCtx(i.context, msg, "error", err) } }) - - return db, nil } diff --git a/cmd/accumulated/run/types_gen.go b/cmd/accumulated/run/types_gen.go index 6ebf2071c..39165c307 100644 --- a/cmd/accumulated/run/types_gen.go +++ b/cmd/accumulated/run/types_gen.go @@ -30,6 +30,11 @@ import ( ) type BadgerStorage struct { + Path string `json:"path,omitempty" form:"path" query:"path" validate:"required"` + Version int64 `json:"version,omitempty" form:"version" query:"version" validate:"required"` +} + +type BoltStorage struct { Path string `json:"path,omitempty" form:"path" query:"path" validate:"required"` } @@ -96,6 +101,10 @@ type EventsService struct { Partition string `json:"partition,omitempty" form:"partition" query:"partition" validate:"required"` } +type ExpBlockDBStorage struct { + Path string `json:"path,omitempty" form:"path" query:"path" validate:"required"` +} + type FaucetService struct { Account *url.URL `json:"account,omitempty" form:"account" query:"account" validate:"required"` SigningKey PrivateKey `json:"signingKey,omitempty" form:"signingKey" query:"signingKey" validate:"required"` @@ -144,6 +153,10 @@ type Instrumentation struct { HttpListener } +type LevelDBStorage struct { + Path string `json:"path,omitempty" form:"path" query:"path" validate:"required"` +} + type Logging struct { Format string `json:"format,omitempty" form:"format" query:"format" validate:"required"` Color *bool `json:"color,omitempty" form:"color" query:"color" validate:"required"` @@ -228,6 +241,8 @@ type TransientPrivateKey struct { func (*BadgerStorage) Type() StorageType { return StorageTypeBadger } +func (*BoltStorage) Type() StorageType { return StorageTypeBolt } + func (*CometNodeKeyFile) Type() PrivateKeyType { return PrivateKeyTypeCometNodeKeyFile } func (*CometPrivValFile) Type() PrivateKeyType { return PrivateKeyTypeCometPrivValFile } @@ -242,12 +257,16 @@ func (*DevnetConfiguration) Type() ConfigurationType { return ConfigurationTypeD func (*EventsService) Type() ServiceType { return ServiceTypeEvents } +func (*ExpBlockDBStorage) Type() StorageType { return StorageTypeExpBlockDB } + func (*FaucetService) Type() ServiceType { return ServiceTypeFaucet } func (*GatewayConfiguration) Type() ConfigurationType { return ConfigurationTypeGateway } func (*HttpService) Type() ServiceType { return ServiceTypeHttp } +func (*LevelDBStorage) Type() StorageType { return StorageTypeLevelDB } + func (*MemoryStorage) Type() StorageType { return StorageTypeMemory } func (*MetricsService) Type() ServiceType { return ServiceTypeMetrics } @@ -274,12 +293,23 @@ func (v *BadgerStorage) Copy() *BadgerStorage { u := new(BadgerStorage) u.Path = v.Path + u.Version = v.Version return u } func (v *BadgerStorage) CopyAsInterface() interface{} { return v.Copy() } +func (v *BoltStorage) Copy() *BoltStorage { + u := new(BoltStorage) + + u.Path = v.Path + + return u +} + +func (v *BoltStorage) CopyAsInterface() interface{} { return v.Copy() } + func (v *CometNodeKeyFile) Copy() *CometNodeKeyFile { u := new(CometNodeKeyFile) @@ -461,6 +491,16 @@ func (v *EventsService) Copy() *EventsService { func (v *EventsService) CopyAsInterface() interface{} { return v.Copy() } +func (v *ExpBlockDBStorage) Copy() *ExpBlockDBStorage { + u := new(ExpBlockDBStorage) + + u.Path = v.Path + + return u +} + +func (v *ExpBlockDBStorage) CopyAsInterface() interface{} { return v.Copy() } + func (v *FaucetService) Copy() *FaucetService { u := new(FaucetService) @@ -589,6 +629,16 @@ func (v *Instrumentation) Copy() *Instrumentation { func (v *Instrumentation) CopyAsInterface() interface{} { return v.Copy() } +func (v *LevelDBStorage) Copy() *LevelDBStorage { + u := new(LevelDBStorage) + + u.Path = v.Path + + return u +} + +func (v *LevelDBStorage) CopyAsInterface() interface{} { return v.Copy() } + func (v *Logging) Copy() *Logging { u := new(Logging) @@ -811,6 +861,17 @@ func (v *BadgerStorage) Equal(u *BadgerStorage) bool { if !(v.Path == u.Path) { return false } + if !(v.Version == u.Version) { + return false + } + + return true +} + +func (v *BoltStorage) Equal(u *BoltStorage) bool { + if !(v.Path == u.Path) { + return false + } return true } @@ -1048,6 +1109,14 @@ func (v *EventsService) Equal(u *EventsService) bool { return true } +func (v *ExpBlockDBStorage) Equal(u *ExpBlockDBStorage) bool { + if !(v.Path == u.Path) { + return false + } + + return true +} + func (v *FaucetService) Equal(u *FaucetService) bool { switch { case v.Account == u.Account: @@ -1195,6 +1264,14 @@ func (v *Instrumentation) Equal(u *Instrumentation) bool { return true } +func (v *LevelDBStorage) Equal(u *LevelDBStorage) bool { + if !(v.Path == u.Path) { + return false + } + + return true +} + func (v *Logging) Equal(u *Logging) bool { if !(v.Format == u.Format) { return false @@ -1625,6 +1702,22 @@ func (v *RouterService) UnmarshalFieldsFrom(reader *encoding.Reader) error { } func (v *BadgerStorage) MarshalJSON() ([]byte, error) { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + Version int64 `json:"version,omitempty"` + }{} + u.Type = v.Type() + if !(len(v.Path) == 0) { + u.Path = v.Path + } + if !(v.Version == 0) { + u.Version = v.Version + } + return json.Marshal(&u) +} + +func (v *BoltStorage) MarshalJSON() ([]byte, error) { u := struct { Type StorageType `json:"type"` Path string `json:"path,omitempty"` @@ -1842,6 +1935,18 @@ func (v *EventsService) MarshalJSON() ([]byte, error) { return json.Marshal(&u) } +func (v *ExpBlockDBStorage) MarshalJSON() ([]byte, error) { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + }{} + u.Type = v.Type() + if !(len(v.Path) == 0) { + u.Path = v.Path + } + return json.Marshal(&u) +} + func (v *FaucetService) MarshalJSON() ([]byte, error) { u := struct { Type ServiceType `json:"type"` @@ -2008,6 +2113,18 @@ func (v *Instrumentation) MarshalJSON() ([]byte, error) { return json.Marshal(&u) } +func (v *LevelDBStorage) MarshalJSON() ([]byte, error) { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + }{} + u.Type = v.Type() + if !(len(v.Path) == 0) { + u.Path = v.Path + } + return json.Marshal(&u) +} + func (v *Logging) MarshalJSON() ([]byte, error) { u := struct { Format string `json:"format,omitempty"` @@ -2237,6 +2354,26 @@ func (v *TransientPrivateKey) MarshalJSON() ([]byte, error) { } func (v *BadgerStorage) UnmarshalJSON(data []byte) error { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + Version int64 `json:"version,omitempty"` + }{} + u.Type = v.Type() + u.Path = v.Path + u.Version = v.Version + if err := json.Unmarshal(data, &u); err != nil { + return err + } + if !(v.Type() == u.Type) { + return fmt.Errorf("field Type: not equal: want %v, got %v", v.Type(), u.Type) + } + v.Path = u.Path + v.Version = u.Version + return nil +} + +func (v *BoltStorage) UnmarshalJSON(data []byte) error { u := struct { Type StorageType `json:"type"` Path string `json:"path,omitempty"` @@ -2511,6 +2648,23 @@ func (v *EventsService) UnmarshalJSON(data []byte) error { return nil } +func (v *ExpBlockDBStorage) UnmarshalJSON(data []byte) error { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + }{} + u.Type = v.Type() + u.Path = v.Path + if err := json.Unmarshal(data, &u); err != nil { + return err + } + if !(v.Type() == u.Type) { + return fmt.Errorf("field Type: not equal: want %v, got %v", v.Type(), u.Type) + } + v.Path = u.Path + return nil +} + func (v *FaucetService) UnmarshalJSON(data []byte) error { u := struct { Type ServiceType `json:"type"` @@ -2714,6 +2868,23 @@ func (v *Instrumentation) UnmarshalJSON(data []byte) error { return nil } +func (v *LevelDBStorage) UnmarshalJSON(data []byte) error { + u := struct { + Type StorageType `json:"type"` + Path string `json:"path,omitempty"` + }{} + u.Type = v.Type() + u.Path = v.Path + if err := json.Unmarshal(data, &u); err != nil { + return err + } + if !(v.Type() == u.Type) { + return fmt.Errorf("field Type: not equal: want %v, got %v", v.Type(), u.Type) + } + v.Path = u.Path + return nil +} + func (v *Logging) UnmarshalJSON(data []byte) error { u := struct { Format string `json:"format,omitempty"` diff --git a/cmd/accumulated/run/unions_gen.go b/cmd/accumulated/run/unions_gen.go index 0fbd4ab5a..ca15f2ec5 100644 --- a/cmd/accumulated/run/unions_gen.go +++ b/cmd/accumulated/run/unions_gen.go @@ -18,6 +18,12 @@ func NewStorage(typ StorageType) (Storage, error) { switch typ { case StorageTypeBadger: return new(BadgerStorage), nil + case StorageTypeBolt: + return new(BoltStorage), nil + case StorageTypeExpBlockDB: + return new(ExpBlockDBStorage), nil + case StorageTypeLevelDB: + return new(LevelDBStorage), nil case StorageTypeMemory: return new(MemoryStorage), nil } @@ -36,6 +42,24 @@ func EqualStorage(a, b Storage) bool { } b, ok := b.(*BadgerStorage) return ok && a.Equal(b) + case *BoltStorage: + if a == nil { + return b == nil + } + b, ok := b.(*BoltStorage) + return ok && a.Equal(b) + case *ExpBlockDBStorage: + if a == nil { + return b == nil + } + b, ok := b.(*ExpBlockDBStorage) + return ok && a.Equal(b) + case *LevelDBStorage: + if a == nil { + return b == nil + } + b, ok := b.(*LevelDBStorage) + return ok && a.Equal(b) case *MemoryStorage: if a == nil { return b == nil @@ -51,6 +75,12 @@ func CopyStorage(v Storage) Storage { switch v := v.(type) { case *BadgerStorage: return v.Copy() + case *BoltStorage: + return v.Copy() + case *ExpBlockDBStorage: + return v.Copy() + case *LevelDBStorage: + return v.Copy() case *MemoryStorage: return v.Copy() default: diff --git a/internal/database/database.go b/internal/database/database.go index 77c9302a6..ab8ac7fc1 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -58,7 +58,7 @@ func OpenBadger(filepath string, logger log.Logger) (*Database, error) { } func OpenLevelDB(filepath string, logger log.Logger) (*Database, error) { - store, err := leveldb.OpenFile(filepath) + store, err := leveldb.Open(filepath) if err != nil { return nil, err } diff --git a/pkg/database/keyvalue/leveldb/changeset_test.go b/pkg/database/keyvalue/leveldb/changeset_test.go index 8d5f534fd..1ff1bd3a8 100644 --- a/pkg/database/keyvalue/leveldb/changeset_test.go +++ b/pkg/database/keyvalue/leveldb/changeset_test.go @@ -16,7 +16,7 @@ import ( func open(t testing.TB) kvtest.Opener { dir := t.TempDir() return func() (keyvalue.Beginner, error) { - return OpenFile(dir) + return Open(dir) } } diff --git a/pkg/database/keyvalue/leveldb/database.go b/pkg/database/keyvalue/leveldb/database.go index d2f131076..ba4e3d23d 100644 --- a/pkg/database/keyvalue/leveldb/database.go +++ b/pkg/database/keyvalue/leveldb/database.go @@ -27,7 +27,7 @@ type opts struct { type Option func(*opts) error -func OpenFile(filepath string, o ...Option) (*Database, error) { +func Open(filepath string, o ...Option) (*Database, error) { // Make sure all directories exist err := os.MkdirAll(filepath, 0700) if err != nil { diff --git a/tools/cmd/debug/db_common.go b/tools/cmd/debug/db_common.go index 3e373c30d..cea2d4f26 100644 --- a/tools/cmd/debug/db_common.go +++ b/tools/cmd/debug/db_common.go @@ -17,6 +17,7 @@ import ( sv1 "gitlab.com/accumulatenetwork/accumulate/internal/database/snapshot" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/badger" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/block" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/bolt" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/leveldb" "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/remote" @@ -112,7 +113,12 @@ func openDbUrl(arg string, writable bool) (_ dbCloser, _ net.Addr) { return db, nil case "leveldb": - db, err := leveldb.OpenFile(u.Path) + db, err := leveldb.Open(u.Path) + check(err) + return db, nil + + case "block": + db, err := block.Open(u.Path) check(err) return db, nil