diff --git a/.vscode/launch.json b/.vscode/launch.json index 1bc1dec..3c1c5d4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,6 +8,14 @@ "mode": "auto", "program": "${workspaceFolder}", "args": ["status"] + }, + { + "name": "dvs migrate", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}", + "args": ["migrate"] } ] } diff --git a/cmd/migrate.go b/cmd/migrate.go new file mode 100644 index 0000000..70cf858 --- /dev/null +++ b/cmd/migrate.go @@ -0,0 +1,47 @@ +package cmd + +import ( + "dvs/internal/log" + "dvs/internal/migrate" + "fmt" + + "github.com/spf13/cobra" +) + +func runMigrateModule(msg string, migrateFunc func() ([]string, error)) { + log.Print(msg) + filesModified, err := migrateFunc() + log.OverwritePreviousLine() + if err != nil { + log.Print(msg, log.IconFailure) + log.Print(" ", log.ColorRed(err)) + } else if len(filesModified) == 0 { + log.Print(msg, log.IconSuccess, log.ColorGreen("already up to date")) + } else { + log.Print(msg, log.IconSuccess, log.ColorGreen("migrated ", fmt.Sprint(len(filesModified)), " files")) + for _, file := range filesModified { + log.Print(" ", log.ColorFile(file)) + } + } +} + +func runMigrateCmd(cmd *cobra.Command, args []string) { + runMigrateModule("Migrating local metadata...", migrate.MigrateMetaFiles) + runMigrateModule("Migrating files in storage...", migrate.MigrateStorageFiles) + + log.Print("\nMigration complete!") +} + +func getMigrateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrates data to the latest format", + Long: "Migrates data to the latest format. This includes the meta files and the storage.", + PreRun: func(cmd *cobra.Command, args []string) { + log.PrintLogo() + }, + Run: runMigrateCmd, + } + + return cmd +} diff --git a/cmd/root.go b/cmd/root.go index 55abaf1..2fe38ab 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -38,6 +38,7 @@ func getRootCmd() *cobra.Command { cmd.AddCommand(getAddCmd()) cmd.AddCommand(getGetCmd()) cmd.AddCommand(getRemoveCmd()) + cmd.AddCommand(getMigrateCmd()) return cmd } diff --git a/internal/log/pretty.go b/internal/log/pretty.go index 390dfcc..0f8cdb6 100644 --- a/internal/log/pretty.go +++ b/internal/log/pretty.go @@ -15,6 +15,9 @@ var ColorYellow = color.New(color.FgYellow).Sprint var ColorFaint = color.New(color.Faint).Sprint var ColorBold = color.New(color.Bold).Sprint var ColorFile = color.New(color.Faint, color.Bold).Sprint +var IconSuccess = ColorGreen("✔") +var IconFailure = ColorRed("✘") +var IconWarning = ColorYellow("⚠") var logOut io.Writer = os.Stdout diff --git a/internal/meta/file.go b/internal/meta/file.go index 97b7336..bbc786a 100644 --- a/internal/meta/file.go +++ b/internal/meta/file.go @@ -40,7 +40,7 @@ func Save(metadata Metadata, path string, dry bool) (err error) { return nil } -// Loads a metadata file +// Loads the metadata file for a given file, erroring if it doesn't exist or if it can't be decoded func Load(path string) (metadata Metadata, err error) { metadataFile, err := os.Open(path + FileExtension) if err != nil { diff --git a/internal/migrate/main.go b/internal/migrate/main.go new file mode 100644 index 0000000..9aef0d9 --- /dev/null +++ b/internal/migrate/main.go @@ -0,0 +1,6 @@ +package migrate + +func MigrateToLatest() { + MigrateMetaFiles() + MigrateStorageFiles() +} diff --git a/internal/migrate/meta.go b/internal/migrate/meta.go new file mode 100644 index 0000000..b261047 --- /dev/null +++ b/internal/migrate/meta.go @@ -0,0 +1,102 @@ +package migrate + +import ( + "dvs/internal/git" + "dvs/internal/meta" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// Attempts to migrate a metafile format from version 1 to the latest, returns true if anything was migrated +func migrateMetaFormatV1(path string) (actionTaken bool, err error) { + type LegacyMetadata struct { + FileHash string `json:"blake3_checksum"` + FileSize uint64 `json:"file_size_bytes"` + Timestamp time.Time `json:"timestamp"` + Message string `json:"message"` + User string `json:"user"` + } + + metadataFile, err := os.Open(path) + if err != nil { + return false, err + } + + var legacyMetadata LegacyMetadata + err = json.NewDecoder(metadataFile).Decode(&legacyMetadata) + if err != nil { + return false, err + } + + // Map the contents to the new format + metadata := meta.Metadata{ + FileHash: legacyMetadata.FileHash, + FileSize: legacyMetadata.FileSize, + Timestamp: legacyMetadata.Timestamp, + Message: legacyMetadata.Message, + SavedBy: legacyMetadata.User, + } + + // Save the new format + meta.Save(metadata, path, false) + + return true, nil +} + +// Migrate a single meta file to the latest format, returns true if anything was migrated +func migrateMetaFile(path string) (actionTaken bool, errs error) { + // Ensure the file has the correct extension + ext := filepath.Ext(path) + if ext != meta.FileExtension { + newPath := strings.TrimSuffix(path, ext) + meta.FileExtension + err := os.Rename(path, newPath) + if err != nil { + return false, err + } + + path = newPath + actionTaken = true + } + + // Ensure the file has the correct content structure + pathNoExt := strings.TrimSuffix(path, meta.FileExtension) + _, err := meta.Load(pathNoExt) + if err != nil { + _, err = migrateMetaFormatV1(path) + if err != nil { + return false, err + } + + actionTaken = true + } + + return actionTaken, err +} + +// Migrates local meta files to the latest format, returning a list of files that were modified +func MigrateMetaFiles() (filesModified []string, err error) { + // Iterate over all files in the git repository + repoDir, _ := git.GetNearestRepoDir(".") + filepath.WalkDir(repoDir, func(path string, d fs.DirEntry, _ error) error { + // TODO respect gitignore? + + // Check if the file is a meta file of some format + if filepath.Ext(path) == ".dvsmeta" || filepath.Ext(path) == ".dvs" { + fileWasMigrated, err := migrateMetaFile(path) + if err != nil { + return err + } + if fileWasMigrated { + filesModified = append(filesModified, path) + } + } + + return nil + }) + + return filesModified, nil +} diff --git a/internal/migrate/storage.go b/internal/migrate/storage.go new file mode 100644 index 0000000..5df85dc --- /dev/null +++ b/internal/migrate/storage.go @@ -0,0 +1,88 @@ +package migrate + +import ( + "dvs/internal/config" + "dvs/internal/git" + "dvs/internal/storage" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func migrateStorageFile(storageDir string, path string) (modified bool, err error) { + // Ensure the file has the correct extension + ext := filepath.Ext(path) + if ext != "" { + newPath := strings.TrimSuffix(path, ext) + err := os.Rename(path, newPath) + if err != nil { + return false, err + } + + path = newPath + modified = true + } + + // Ensure the file is not at the root level + if filepath.Dir(path) == storageDir { + // Create new directory + newDir := storage.GetStoragePath(storageDir, filepath.Base(path)) + err := os.MkdirAll(newDir, storage.StorageDirPermissions) + if err != nil { + return false, err + } + + // Move to correct location + newPath := storage.GetStoragePath(storageDir, filepath.Base(path)) + err = os.Rename(path, newPath) + + modified = true + + if os.IsExist(err) { + // File already exists, delete the old one + err = os.Remove(path) + if err != nil { + return modified, err + } + } else if err != nil { + return modified, err + } + } + + return modified, nil +} + +// Migrates storage files to the latest format, returning a list of files that were modified +func MigrateStorageFiles() (modifiedFiles []string, err error) { + repoDir, _ := git.GetNearestRepoDir(".") + config, err := config.Read(repoDir) + if err != nil { + return nil, err + } + + // Iterate over all files in the storage directory + err = filepath.WalkDir(config.StorageDir, func(path string, d fs.DirEntry, _ error) error { + // Don't migrate the directories + if d.IsDir() { + return nil + } + + // TODO respect gitignore? + modified, err := migrateStorageFile(config.StorageDir, path) + if err != nil { + return err + } + + if modified { + modifiedFiles = append(modifiedFiles, path) + } + + return nil + }) + if err != nil { + return nil, err + } + + return modifiedFiles, nil +} diff --git a/internal/storage/add.go b/internal/storage/add.go index a7112cf..d829238 100644 --- a/internal/storage/add.go +++ b/internal/storage/add.go @@ -32,7 +32,7 @@ func Add(localPath string, storageDir string, gitDir string, message string, dry }) // Get storage path - dstPath := getStoragePath(storageDir, fileHash) + dstPath := GetStoragePath(storageDir, fileHash) // Copy the file to the storage directory // if the destination already exists, skip copying diff --git a/internal/storage/copy.go b/internal/storage/copy.go index 8f1ccc3..58c3e09 100644 --- a/internal/storage/copy.go +++ b/internal/storage/copy.go @@ -56,7 +56,7 @@ func Copy(srcPath string, destPath string, dry bool) error { }) // Ensure destination exists - err = os.MkdirAll(filepath.Dir(destPath), storageDirPermissions) + err = os.MkdirAll(filepath.Dir(destPath), StorageDirPermissions) if err != nil { return err } diff --git a/internal/storage/get.go b/internal/storage/get.go index bceb0e9..5305855 100644 --- a/internal/storage/get.go +++ b/internal/storage/get.go @@ -22,7 +22,7 @@ func Get(localPath string, storageDir string, gitDir string, dry bool) error { } // Get storage path - storagePath := getStoragePath(storageDir, metadata.FileHash) + storagePath := GetStoragePath(storageDir, metadata.FileHash) // Check if file is already present locally _, err = os.Stat(localPath) diff --git a/internal/storage/get_test.go b/internal/storage/get_test.go index aabad0d..032781d 100644 --- a/internal/storage/get_test.go +++ b/internal/storage/get_test.go @@ -45,7 +45,7 @@ func TestGetNoLongerInStorage(t *testing.T) { } // Remove file from storage manually - err = os.Remove(getStoragePath(tempDir, hash)) + err = os.Remove(GetStoragePath(tempDir, hash)) if err != nil { t.Fatal(err) } diff --git a/internal/storage/init.go b/internal/storage/init.go index 3ed014a..b691919 100644 --- a/internal/storage/init.go +++ b/internal/storage/init.go @@ -25,7 +25,7 @@ func Init(rootDir string, storageDir string) error { fileInfo, err := os.Stat(storageDir) if err != nil { // Create storage dir and necessary parents - err = os.MkdirAll(storageDir, storageDirPermissions) + err = os.MkdirAll(storageDir, StorageDirPermissions) if err != nil { log.Print(log.ColorRed("✘"), "Failed to create storage directory", log.ColorFile(storageDir)) log.JsonLogger.Issues = append(log.JsonLogger.Issues, log.JsonIssue{ @@ -38,7 +38,7 @@ func Init(rootDir string, storageDir string) error { } // Set storage dir permissions - err = os.Chmod(storageDir, storageDirPermissions) + err = os.Chmod(storageDir, StorageDirPermissions) if err != nil { log.Print(log.ColorRed("✘"), "Failed to set storage directory permissions", log.ColorFile(storageDir)) log.JsonLogger.Issues = append(log.JsonLogger.Issues, log.JsonIssue{ diff --git a/internal/storage/main.go b/internal/storage/main.go index e83e3dc..8ef98b6 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -6,11 +6,11 @@ import ( ) var ( - storageDirPermissions = fs.FileMode(0777) + StorageDirPermissions = fs.FileMode(0777) storageFilePermissions = fs.FileMode(0666) ) -func getStoragePath(storageDir string, fileHash string) string { +func GetStoragePath(storageDir string, fileHash string) string { firstHashSegment := fileHash[:2] secondHashSegment := fileHash[2:] return filepath.Join(storageDir, firstHashSegment, secondHashSegment) diff --git a/internal/storage/remove.go b/internal/storage/remove.go index bc4b9cc..1bba3f5 100644 --- a/internal/storage/remove.go +++ b/internal/storage/remove.go @@ -17,7 +17,7 @@ func Remove(path string, conf config.Config, gitDir string, dry bool) error { } // Get storage path - storagePath := getStoragePath(conf.StorageDir, metadata.FileHash) + storagePath := GetStoragePath(conf.StorageDir, metadata.FileHash) // Remove file from storage if !dry { diff --git a/testing/10G.dvs b/testing/10G.dvs index c0054c9..9fabcd8 100644 --- a/testing/10G.dvs +++ b/testing/10G.dvs @@ -1,7 +1,7 @@ { "blake3_checksum": "28960eef7d587ab6d1627b7efe30c7a07ce2dce4871d339fdfb607cb0776e064", "file_size_bytes": 10737418240, - "timestamp": "2023-10-11T12:49:39.475255693-04:00", + "timestamp": "2023-10-11T11:47:21.735488265-04:00", "message": "", "saved_by": "andriygm" } \ No newline at end of file diff --git a/testing/10G.dvsmeta b/testing/10G.dvsmeta deleted file mode 100644 index 9fabcd8..0000000 --- a/testing/10G.dvsmeta +++ /dev/null @@ -1,7 +0,0 @@ -{ - "blake3_checksum": "28960eef7d587ab6d1627b7efe30c7a07ce2dce4871d339fdfb607cb0776e064", - "file_size_bytes": 10737418240, - "timestamp": "2023-10-11T11:47:21.735488265-04:00", - "message": "", - "saved_by": "andriygm" -} \ No newline at end of file diff --git a/testing/1G.dvs b/testing/1G.dvs index c1f49fb..47a6ec8 100644 --- a/testing/1G.dvs +++ b/testing/1G.dvs @@ -1,7 +1,7 @@ { "blake3_checksum": "94b4ec39d8d42ebda685fbb5429e8ab0086e65245e750142c1eea36a26abc24d", "file_size_bytes": 1073741824, - "timestamp": "2023-10-11T12:49:20.077594173-04:00", + "timestamp": "2023-10-11T11:47:21.736802714-04:00", "message": "", "saved_by": "andriygm" } \ No newline at end of file diff --git a/testing/1G.dvsmeta b/testing/1G.dvsmeta deleted file mode 100644 index 47a6ec8..0000000 --- a/testing/1G.dvsmeta +++ /dev/null @@ -1,7 +0,0 @@ -{ - "blake3_checksum": "94b4ec39d8d42ebda685fbb5429e8ab0086e65245e750142c1eea36a26abc24d", - "file_size_bytes": 1073741824, - "timestamp": "2023-10-11T11:47:21.736802714-04:00", - "message": "", - "saved_by": "andriygm" -} \ No newline at end of file diff --git a/testing/266-536x354.jpg.dvs b/testing/266-536x354.jpg.dvs index ca3d6bf..7a48e47 100644 --- a/testing/266-536x354.jpg.dvs +++ b/testing/266-536x354.jpg.dvs @@ -1,7 +1,7 @@ { "blake3_checksum": "8ea93e919458fe8db0c7079c775f975af6d5a654daf9dcf3651e60a5a94879d8", "file_size_bytes": 22600, - "timestamp": "2023-10-11T12:48:34.89499103-04:00", + "timestamp": "2023-10-11T11:47:21.737105368-04:00", "message": "", "saved_by": "andriygm" } \ No newline at end of file diff --git a/testing/266-536x354.jpg.dvsmeta b/testing/266-536x354.jpg.dvsmeta deleted file mode 100644 index 7a48e47..0000000 --- a/testing/266-536x354.jpg.dvsmeta +++ /dev/null @@ -1,7 +0,0 @@ -{ - "blake3_checksum": "8ea93e919458fe8db0c7079c775f975af6d5a654daf9dcf3651e60a5a94879d8", - "file_size_bytes": 22600, - "timestamp": "2023-10-11T11:47:21.737105368-04:00", - "message": "", - "saved_by": "andriygm" -} \ No newline at end of file diff --git a/testing/571-536x354.jpg.dvs b/testing/571-536x354.jpg.dvs index fd73f50..cf7179e 100644 --- a/testing/571-536x354.jpg.dvs +++ b/testing/571-536x354.jpg.dvs @@ -1,7 +1,7 @@ { "blake3_checksum": "800fe626b5b4a066de815d9486893fe549e2af1742558ed4110c6ea424425c42", "file_size_bytes": 23091, - "timestamp": "2023-10-11T12:49:10.692239925-04:00", + "timestamp": "2023-10-11T11:47:21.737655146-04:00", "message": "", "saved_by": "andriygm" } \ No newline at end of file diff --git a/testing/571-536x354.jpg.dvsmeta b/testing/571-536x354.jpg.dvsmeta deleted file mode 100644 index cf7179e..0000000 --- a/testing/571-536x354.jpg.dvsmeta +++ /dev/null @@ -1,7 +0,0 @@ -{ - "blake3_checksum": "800fe626b5b4a066de815d9486893fe549e2af1742558ed4110c6ea424425c42", - "file_size_bytes": 23091, - "timestamp": "2023-10-11T11:47:21.737655146-04:00", - "message": "", - "saved_by": "andriygm" -} \ No newline at end of file