Merge pull request #57 from A2-ai/feat/storage-subdirs

Segment storage files into subdirs

internal/meta/parse.go

@@ -10,7 +10,7 @@ import (
 	"golang.org/x/exp/slices"
 )
 
-var FileExtension = ".dvsmeta"
+var FileExtension = ".dvs"
 
 // Gets a list of all meta file paths in the directory recursively
 func GetAllMetaFiles(dir string) (metaFiles []string, err error) {

internal/storage/add.go

@@ -8,7 +8,6 @@ import (
 	"dvs/internal/utils"
 	"os"
 	"os/user"
-	"path/filepath"
 	"time"
 )
 
@@ -32,7 +31,8 @@ func Add(localPath string, storageDir string, gitDir string, message string, dry
 		Path:   localPath,
 	})
 
-	dstPath := filepath.Join(storageDir, fileHash) + FileExtension
+	// Get storage path
+	dstPath := getStoragePath(storageDir, fileHash)
 
 	// Copy the file to the storage directory
 	// if the destination already exists, skip copying

internal/storage/get.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 )
 
@@ -23,7 +22,7 @@ func Get(localPath string, storageDir string, gitDir string, dry bool) error {
 	}
 
 	// Get storage path
-	storagePath := filepath.Join(storageDir, metadata.FileHash) + FileExtension
+	storagePath := getStoragePath(storageDir, metadata.FileHash)
 
 	// Check if file is already present locally
 	_, err = os.Stat(localPath)

internal/storage/get_test.go

@@ -45,7 +45,7 @@ func TestGetNoLongerInStorage(t *testing.T) {
 	}
 
 	// Remove file from storage manually
-	err = os.Remove(filepath.Join(tempDir, hash) + FileExtension)
+	err = os.Remove(getStoragePath(tempDir, hash))
 	if err != nil {
 		t.Fatal(err)
 	}

internal/storage/main.go

@@ -1,9 +1,17 @@
 package storage
 
-import "io/fs"
+import (
+	"io/fs"
+	"path/filepath"
+)
 
 var (
-	FileExtension          = ".dvsfile"
 	storageDirPermissions  = fs.FileMode(0777)
 	storageFilePermissions = fs.FileMode(0666)
 )
+
+func getStoragePath(storageDir string, fileHash string) string {
+	firstHashSegment := fileHash[:2]
+	secondHashSegment := fileHash[2:]
+	return filepath.Join(storageDir, firstHashSegment, secondHashSegment)
+}

internal/storage/remove.go

@@ -6,7 +6,6 @@ import (
 	"dvs/internal/log"
 	"dvs/internal/meta"
 	"os"
-	"path/filepath"
 )
 
 // Remove a file from storage
@@ -18,7 +17,7 @@ func Remove(path string, conf config.Config, gitDir string, dry bool) error {
 	}
 
 	// Get storage path
-	storagePath := filepath.Join(conf.StorageDir, metadata.FileHash) + FileExtension
+	storagePath := getStoragePath(conf.StorageDir, metadata.FileHash)
 
 	// Remove file from storage
 	if !dry {

testing/10G.dvs

@@ -0,0 +1,7 @@
+{
+  "blake3_checksum": "28960eef7d587ab6d1627b7efe30c7a07ce2dce4871d339fdfb607cb0776e064",
+  "file_size_bytes": 10737418240,
+  "timestamp": "2023-10-11T12:49:39.475255693-04:00",
+  "message": "",
+  "saved_by": "andriygm"
+}

testing/1G.dvs

@@ -0,0 +1,7 @@
+{
+  "blake3_checksum": "94b4ec39d8d42ebda685fbb5429e8ab0086e65245e750142c1eea36a26abc24d",
+  "file_size_bytes": 1073741824,
+  "timestamp": "2023-10-11T12:49:20.077594173-04:00",
+  "message": "",
+  "saved_by": "andriygm"
+}

testing/266-536x354.jpg.dvs

@@ -0,0 +1,7 @@
+{
+  "blake3_checksum": "8ea93e919458fe8db0c7079c775f975af6d5a654daf9dcf3651e60a5a94879d8",
+  "file_size_bytes": 22600,
+  "timestamp": "2023-10-11T12:48:34.89499103-04:00",
+  "message": "",
+  "saved_by": "andriygm"
+}

testing/571-536x354.jpg.dvs

@@ -0,0 +1,7 @@
+{
+  "blake3_checksum": "800fe626b5b4a066de815d9486893fe549e2af1742558ed4110c6ea424425c42",
+  "file_size_bytes": 23091,
+  "timestamp": "2023-10-11T12:49:10.692239925-04:00",
+  "message": "",
+  "saved_by": "andriygm"
+}