diff --git a/CHANGELOG.md b/CHANGELOG.md index c9e648fd587..10c74cceeff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Changelog for NeoFS Node - Histogram metrics for RPC and engine operations (#2351) - SN's version is announced via the attributes automatically but can be overwritten explicitly (#2455) - New storage component for small objects named Peapod (#2453) +- New `blobovnicza-to-peapod` tool providing blobovnicza-to-peapod data migration (#2453) ### Fixed - `neo-go` RPC connection loss handling (#1337) @@ -72,6 +73,15 @@ Docker images now contain a single executable file and SSL certificates only. `neofs-cli control healthcheck` exit code is `0` only for "READY" state. +To migrate data from Blobovnicza trees to Peapods: +```shell +$ blobovnicza-to-peapod -config +``` +For any shard, the data from the configured Blobovnicza tree is copied into +a created Peapod file named `peapod.db` in the directory where the tree is +located. For example, `/neofs/data/blobovcniza/*` -> `/neofs/data/peapod.db`. +Notice that existing Blobovnicza trees are untouched. + ## [0.37.0] - 2023-06-15 - Sogado ### Added diff --git a/cmd/blobovnicza-to-peapod/main.go b/cmd/blobovnicza-to-peapod/main.go new file mode 100644 index 00000000000..1104ff17c4d --- /dev/null +++ b/cmd/blobovnicza-to-peapod/main.go @@ -0,0 +1,95 @@ +package main + +import ( + "flag" + "io/fs" + "log" + "path/filepath" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" + engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine" + shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" + blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod" +) + +func main() { + nodeCfgPath := flag.String("config", "", "Path to storage node's YAML configuration file") + + flag.Parse() + + if *nodeCfgPath == "" { + log.Fatal("missing storage node config flag") + } + + appCfg := config.New(config.Prm{}, config.WithConfigFile(*nodeCfgPath)) + + err := engineconfig.IterateShards(appCfg, false, func(sc *shardconfig.Config) error { + log.Println("processing shard...") + + var bbcz common.Storage + var perm fs.FileMode + storagesCfg := sc.BlobStor().Storages() + + for i := range storagesCfg { + if storagesCfg[i].Type() == blobovniczatree.Type { + bbczCfg := blobovniczaconfig.From((*config.Config)(storagesCfg[i])) + + perm = storagesCfg[i].Perm() + bbcz = blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(storagesCfg[i].Path()), + blobovniczatree.WithPermissions(storagesCfg[i].Perm()), + blobovniczatree.WithBlobovniczaSize(bbczCfg.Size()), + blobovniczatree.WithBlobovniczaShallowDepth(bbczCfg.ShallowDepth()), + blobovniczatree.WithBlobovniczaShallowWidth(bbczCfg.ShallowWidth()), + blobovniczatree.WithOpenedCacheSize(bbczCfg.OpenedCacheSize())) + + break + } + } + + if bbcz == nil { + log.Println("Blobovnicza is not configured for the current shard, going to next one...") + return nil + } + + bbczPath := bbcz.Path() + if !filepath.IsAbs(bbczPath) { + log.Fatalf("Blobobvnicza tree path '%s' is not absolute, make it like this in the config file first\n", bbczPath) + } + + ppdPath := filepath.Join(filepath.Dir(bbcz.Path()), "peapod.db") + ppd := peapod.New(ppdPath, perm) + + var compressCfg compression.Config + compressCfg.Enabled = sc.Compress() + compressCfg.UncompressableContentTypes = sc.UncompressableContentTypes() + + err := compressCfg.Init() + if err != nil { + log.Fatal("init compression config for the current shard: ", err) + } + + bbcz.SetCompressor(&compressCfg) + ppd.SetCompressor(&compressCfg) + + log.Printf("migrating data from Blobovnicza tree '%s' to Peapod '%s'...\n", bbcz.Path(), ppd.Path()) + + err = common.Copy(ppd, bbcz) + if err != nil { + log.Fatal("migration failed: ", err) + } + + log.Println("data successfully migrated in the current shard, going to the next one...") + + return nil + }) + if err != nil { + log.Fatal(err) + } + + log.Println("data successfully migrated in all shards, you may now re-configure node to work with Peapod") +} diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index b66b4120057..6a6ecd9a910 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -1,6 +1,10 @@ package common -import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" +) // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. @@ -23,3 +27,58 @@ type Storage interface { Delete(DeletePrm) (DeleteRes, error) Iterate(IteratePrm) (IterateRes, error) } + +// Copy copies all objects from source Storage into the destination one. If any +// object cannot be stored, Copy immediately fails. +func Copy(dst, src Storage) error { + err := src.Open(true) + if err != nil { + return fmt.Errorf("open source sub-storage: %w", err) + } + + defer func() { _ = src.Close() }() + + err = src.Init() + if err != nil { + return fmt.Errorf("initialize source sub-storage: %w", err) + } + + err = dst.Open(false) + if err != nil { + return fmt.Errorf("open destination sub-storage: %w", err) + } + + defer func() { _ = dst.Close() }() + + err = dst.Init() + if err != nil { + return fmt.Errorf("initialize destination sub-storage: %w", err) + } + + _, err = src.Iterate(IteratePrm{ + Handler: func(el IterationElement) error { + exRes, err := dst.Exists(ExistsPrm{ + Address: el.Address, + }) + if err != nil { + return fmt.Errorf("check presence of object %s in the destination sub-storage: %w", el.Address, err) + } else if exRes.Exists { + return nil + } + + _, err = dst.Put(PutPrm{ + Address: el.Address, + RawData: el.ObjectData, + }) + if err != nil { + return fmt.Errorf("put object %s into destination sub-storage: %w", el.Address, err) + } + return nil + }, + }) + if err != nil { + return fmt.Errorf("iterate over source sub-storage: %w", err) + } + + return nil +} diff --git a/pkg/local_object_storage/blobstor/common/storage_test.go b/pkg/local_object_storage/blobstor/common/storage_test.go new file mode 100644 index 00000000000..793862cbb30 --- /dev/null +++ b/pkg/local_object_storage/blobstor/common/storage_test.go @@ -0,0 +1,63 @@ +package common_test + +import ( + "crypto/rand" + "path/filepath" + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestCopy(t *testing.T) { + dir := t.TempDir() + const nObjects = 100 + + src := blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithBlobovniczaShallowWidth(2), + blobovniczatree.WithBlobovniczaShallowDepth(3), + blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")), + ) + + require.NoError(t, src.Open(false)) + require.NoError(t, src.Init()) + + mObjs := make(map[oid.Address][]byte, nObjects) + + for i := 0; i < nObjects; i++ { + addr := oidtest.Address() + data := make([]byte, 32) + rand.Read(data) + mObjs[addr] = data + + _, err := src.Put(common.PutPrm{ + Address: addr, + RawData: data, + }) + require.NoError(t, err) + } + + require.NoError(t, src.Close()) + + dst := peapod.New(filepath.Join(dir, "peapod.db"), 0600) + + err := common.Copy(dst, src) + require.NoError(t, err) + + require.NoError(t, dst.Open(true)) + t.Cleanup(func() { _ = dst.Close() }) + + _, err = dst.Iterate(common.IteratePrm{ + Handler: func(el common.IterationElement) error { + data, ok := mObjs[el.Address] + require.True(t, ok) + require.Equal(t, data, el.ObjectData) + return nil + }, + }) + require.NoError(t, err) +}