From e9d4e9af3b85e06d0c96c652e0f309186c583b0d Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:03:28 -0400 Subject: [PATCH 1/8] Readd gitignore --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d492e2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Go workspace file +go.work + +# Build file +dvs + +# Test environment +dvs.yaml +testenv/ \ No newline at end of file From 73d8c8b19cdbe5ef9e585d57097db269593c6b91 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:03:28 -0400 Subject: [PATCH 2/8] Readd gitignore --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d492e2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Go workspace file +go.work + +# Build file +dvs + +# Test environment +dvs.yaml +testenv/ \ No newline at end of file From 9abe751d54b28bb9210991a5992d0c8255b0f964 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:49:28 -0400 Subject: [PATCH 3/8] Add some files for testing --- .dvs.yaml | 1 + .gitignore | 16 ++++++++++++---- testing/10G.dvsmeta | 7 +++++++ testing/1G.dvsmeta | 7 +++++++ testing/266-536x354.jpg.dvsmeta | 7 +++++++ testing/571-536x354.jpg.dvsmeta | 7 +++++++ 6 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 .dvs.yaml create mode 100644 testing/10G.dvsmeta create mode 100644 testing/1G.dvsmeta create mode 100644 testing/266-536x354.jpg.dvsmeta create mode 100644 testing/571-536x354.jpg.dvsmeta diff --git a/.dvs.yaml b/.dvs.yaml new file mode 100644 index 0000000..e1125a0 --- /dev/null +++ b/.dvs.yaml @@ -0,0 +1 @@ +storage-dir: /data/dvs/devious diff --git a/.gitignore b/.gitignore index d492e2e..ea1fc64 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,17 @@ # Go workspace file go.work -# Build file +# Binaries dvs -# Test environment -dvs.yaml -testenv/ \ No newline at end of file +# Devious entry +/testing/266-536x354.jpg + +# Devious entry +/testing/10G + +# Devious entry +/testing/1G + +# Devious entry +/testing/571-536x354.jpg \ No newline at end of file diff --git a/testing/10G.dvsmeta b/testing/10G.dvsmeta new file mode 100644 index 0000000..9fabcd8 --- /dev/null +++ b/testing/10G.dvsmeta @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "28960eef7d587ab6d1627b7efe30c7a07ce2dce4871d339fdfb607cb0776e064", + "file_size_bytes": 10737418240, + "timestamp": "2023-10-11T11:47:21.735488265-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/1G.dvsmeta b/testing/1G.dvsmeta new file mode 100644 index 0000000..47a6ec8 --- /dev/null +++ b/testing/1G.dvsmeta @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "94b4ec39d8d42ebda685fbb5429e8ab0086e65245e750142c1eea36a26abc24d", + "file_size_bytes": 1073741824, + "timestamp": "2023-10-11T11:47:21.736802714-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/266-536x354.jpg.dvsmeta b/testing/266-536x354.jpg.dvsmeta new file mode 100644 index 0000000..7a48e47 --- /dev/null +++ b/testing/266-536x354.jpg.dvsmeta @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "8ea93e919458fe8db0c7079c775f975af6d5a654daf9dcf3651e60a5a94879d8", + "file_size_bytes": 22600, + "timestamp": "2023-10-11T11:47:21.737105368-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/571-536x354.jpg.dvsmeta b/testing/571-536x354.jpg.dvsmeta new file mode 100644 index 0000000..cf7179e --- /dev/null +++ b/testing/571-536x354.jpg.dvsmeta @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "800fe626b5b4a066de815d9486893fe549e2af1742558ed4110c6ea424425c42", + "file_size_bytes": 23091, + "timestamp": "2023-10-11T11:47:21.737655146-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file From f2a6d459c9e9a5672f0a1d3857ac3e1929d484cf Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:49:39 -0400 Subject: [PATCH 4/8] Change user to saved_by --- cmd/status.go | 6 +++--- internal/meta/file.go | 2 +- internal/storage/add.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/status.go b/cmd/status.go index bf7c724..3089f36 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -20,7 +20,7 @@ func runStatusCmd(cmd *cobra.Command, args []string) error { FileSize uint64 `json:"fileSize"` FileHash string `json:"fileHash"` Timestamp string `json:"timestamp"` - User string `json:"user"` + SavedBy string `json:"savedBy"` Message string `json:"message"` } @@ -133,7 +133,7 @@ func runStatusCmd(cmd *cobra.Command, args []string) error { log.ColorFile(relPath), "", log.ColorFaint(humanize.Bytes(metadata.FileSize)), "", log.ColorFaint(timestamp), "", - log.ColorFaint(metadata.User), "", + log.ColorFaint(metadata.SavedBy), "", log.ColorFaint(metadata.Message), ) jsonLogger.Files = append(jsonLogger.Files, jsonFileResult{ @@ -142,7 +142,7 @@ func runStatusCmd(cmd *cobra.Command, args []string) error { FileSize: metadata.FileSize, FileHash: metadata.FileHash, Timestamp: timestamp, - User: metadata.User, + SavedBy: metadata.SavedBy, Message: metadata.Message, }) } diff --git a/internal/meta/file.go b/internal/meta/file.go index 588e79d..97b7336 100644 --- a/internal/meta/file.go +++ b/internal/meta/file.go @@ -11,7 +11,7 @@ type Metadata struct { FileSize uint64 `json:"file_size_bytes"` Timestamp time.Time `json:"timestamp"` Message string `json:"message"` - User string `json:"user"` + SavedBy string `json:"saved_by"` } // Creates a metadata file diff --git a/internal/storage/add.go b/internal/storage/add.go index 1374470..6a82f0c 100644 --- a/internal/storage/add.go +++ b/internal/storage/add.go @@ -87,7 +87,7 @@ func Add(localPath string, storageDir string, gitDir string, message string, dry FileHash: fileHash, FileSize: fileSize, Timestamp: time.Now(), - User: userName, + SavedBy: userName, Message: message, } err = meta.Save(metadata, localPath, dry) From fd9cd6f36f8bf10278ccda75ee758ecfe3a91463 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:57:14 -0400 Subject: [PATCH 5/8] Increase time gap for get test --- internal/storage/get_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/storage/get_test.go b/internal/storage/get_test.go index 14874e6..d23be72 100644 --- a/internal/storage/get_test.go +++ b/internal/storage/get_test.go @@ -106,7 +106,7 @@ func TestGetAgainAfterLocalMod(t *testing.T) { } // Wait a bit to ensure modification time is different - time.Sleep(time.Millisecond) + time.Sleep(2 * time.Millisecond) // Modify the file locally err = os.WriteFile(filepath.Join(tempDir, "test.txt"), []byte("test2"), 0644) From ce6b7d558a18d3804639d7318d269e69566f6400 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 11:57:25 -0400 Subject: [PATCH 6/8] Rename .dvs.yaml to dvs.yaml --- .dvs.yaml => dvs.yaml | 0 internal/config/main.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .dvs.yaml => dvs.yaml (100%) diff --git a/.dvs.yaml b/dvs.yaml similarity index 100% rename from .dvs.yaml rename to dvs.yaml diff --git a/internal/config/main.go b/internal/config/main.go index 159e377..e2f0206 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -11,7 +11,7 @@ type Config struct { StorageDir string `yaml:"storage-dir"` } -var ConfigFileName = ".dvs.yaml" +var ConfigFileName = "dvs.yaml" func Read(rootDir string) (Config, error) { // Read the config file From 745b7aa0866e301e2265a9bfb0b94fb2deb84948 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 12:11:23 -0400 Subject: [PATCH 7/8] Add launch config for status command --- .vscode/launch.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1bc1dec --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "dvs status", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}", + "args": ["status"] + } + ] +} From 10a896a8f8567f42abf73d2ce2b3e3dd64519f85 Mon Sep 17 00:00:00 2001 From: Andriy Massimilla Date: Wed, 11 Oct 2023 12:52:08 -0400 Subject: [PATCH 8/8] Rename .dvsmeta to .dvs, segment storage files into subdirs --- internal/meta/parse.go | 2 +- internal/storage/add.go | 4 ++-- internal/storage/get.go | 3 +-- internal/storage/get_test.go | 2 +- internal/storage/main.go | 12 ++++++++++-- internal/storage/remove.go | 3 +-- testing/10G.dvs | 7 +++++++ testing/1G.dvs | 7 +++++++ testing/266-536x354.jpg.dvs | 7 +++++++ testing/571-536x354.jpg.dvs | 7 +++++++ 10 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 testing/10G.dvs create mode 100644 testing/1G.dvs create mode 100644 testing/266-536x354.jpg.dvs create mode 100644 testing/571-536x354.jpg.dvs diff --git a/internal/meta/parse.go b/internal/meta/parse.go index 3cc8f02..b6913e0 100644 --- a/internal/meta/parse.go +++ b/internal/meta/parse.go @@ -10,7 +10,7 @@ import ( "golang.org/x/exp/slices" ) -var FileExtension = ".dvsmeta" +var FileExtension = ".dvs" // Gets a list of all meta file paths in the directory recursively func GetAllMetaFiles(dir string) (metaFiles []string, err error) { diff --git a/internal/storage/add.go b/internal/storage/add.go index 6a82f0c..a7112cf 100644 --- a/internal/storage/add.go +++ b/internal/storage/add.go @@ -8,7 +8,6 @@ import ( "dvs/internal/utils" "os" "os/user" - "path/filepath" "time" ) @@ -32,7 +31,8 @@ func Add(localPath string, storageDir string, gitDir string, message string, dry Path: localPath, }) - dstPath := filepath.Join(storageDir, fileHash) + FileExtension + // Get storage path + dstPath := getStoragePath(storageDir, fileHash) // Copy the file to the storage directory // if the destination already exists, skip copying diff --git a/internal/storage/get.go b/internal/storage/get.go index 673769d..bceb0e9 100644 --- a/internal/storage/get.go +++ b/internal/storage/get.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "os" - "path/filepath" "time" ) @@ -23,7 +22,7 @@ func Get(localPath string, storageDir string, gitDir string, dry bool) error { } // Get storage path - storagePath := filepath.Join(storageDir, metadata.FileHash) + FileExtension + storagePath := getStoragePath(storageDir, metadata.FileHash) // Check if file is already present locally _, err = os.Stat(localPath) diff --git a/internal/storage/get_test.go b/internal/storage/get_test.go index d23be72..aabad0d 100644 --- a/internal/storage/get_test.go +++ b/internal/storage/get_test.go @@ -45,7 +45,7 @@ func TestGetNoLongerInStorage(t *testing.T) { } // Remove file from storage manually - err = os.Remove(filepath.Join(tempDir, hash) + FileExtension) + err = os.Remove(getStoragePath(tempDir, hash)) if err != nil { t.Fatal(err) } diff --git a/internal/storage/main.go b/internal/storage/main.go index e8e3f67..e83e3dc 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -1,9 +1,17 @@ package storage -import "io/fs" +import ( + "io/fs" + "path/filepath" +) var ( - FileExtension = ".dvsfile" storageDirPermissions = fs.FileMode(0777) storageFilePermissions = fs.FileMode(0666) ) + +func getStoragePath(storageDir string, fileHash string) string { + firstHashSegment := fileHash[:2] + secondHashSegment := fileHash[2:] + return filepath.Join(storageDir, firstHashSegment, secondHashSegment) +} diff --git a/internal/storage/remove.go b/internal/storage/remove.go index cdae9bd..bc4b9cc 100644 --- a/internal/storage/remove.go +++ b/internal/storage/remove.go @@ -6,7 +6,6 @@ import ( "dvs/internal/log" "dvs/internal/meta" "os" - "path/filepath" ) // Remove a file from storage @@ -18,7 +17,7 @@ func Remove(path string, conf config.Config, gitDir string, dry bool) error { } // Get storage path - storagePath := filepath.Join(conf.StorageDir, metadata.FileHash) + FileExtension + storagePath := getStoragePath(conf.StorageDir, metadata.FileHash) // Remove file from storage if !dry { diff --git a/testing/10G.dvs b/testing/10G.dvs new file mode 100644 index 0000000..c0054c9 --- /dev/null +++ b/testing/10G.dvs @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "28960eef7d587ab6d1627b7efe30c7a07ce2dce4871d339fdfb607cb0776e064", + "file_size_bytes": 10737418240, + "timestamp": "2023-10-11T12:49:39.475255693-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/1G.dvs b/testing/1G.dvs new file mode 100644 index 0000000..c1f49fb --- /dev/null +++ b/testing/1G.dvs @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "94b4ec39d8d42ebda685fbb5429e8ab0086e65245e750142c1eea36a26abc24d", + "file_size_bytes": 1073741824, + "timestamp": "2023-10-11T12:49:20.077594173-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/266-536x354.jpg.dvs b/testing/266-536x354.jpg.dvs new file mode 100644 index 0000000..ca3d6bf --- /dev/null +++ b/testing/266-536x354.jpg.dvs @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "8ea93e919458fe8db0c7079c775f975af6d5a654daf9dcf3651e60a5a94879d8", + "file_size_bytes": 22600, + "timestamp": "2023-10-11T12:48:34.89499103-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file diff --git a/testing/571-536x354.jpg.dvs b/testing/571-536x354.jpg.dvs new file mode 100644 index 0000000..fd73f50 --- /dev/null +++ b/testing/571-536x354.jpg.dvs @@ -0,0 +1,7 @@ +{ + "blake3_checksum": "800fe626b5b4a066de815d9486893fe549e2af1742558ed4110c6ea424425c42", + "file_size_bytes": 23091, + "timestamp": "2023-10-11T12:49:10.692239925-04:00", + "message": "", + "saved_by": "andriygm" +} \ No newline at end of file