cleanup FileEntry

Signed-off-by: Naoki MATSUMOTO <[email protected]>

Makefile

@@ -15,7 +15,6 @@ ctr-cli:
 
 diff-tools:
 	go build -o diff ./cmd/diff
-	go build -o pack ./cmd/pack
 
 server:
 	go build -o server ./cmd/server
@@ -30,4 +29,4 @@ patch:
 	go build -o patch ./cmd/patch
 
 clean:
-	rm -f snapshotter ctr-cli diff pack server fuse-diff merge patch
+	rm -f snapshotter ctr-cli diff server fuse-diff merge patch

cmd/ctr-cli/convert/convert.go

@@ -234,15 +234,9 @@ func Action(c *cli.Context) error {
 	}
 	defer os.RemoveAll(tempDiffDir)
 
-	logger.Info("generating Di3FS file entries")
-	entry, err := di3fsImage.GenerateDiffFromDir("", tempDir, tempDiffDir, true, false)
-	if err != nil {
-		return fmt.Errorf("failed to generate diffs: %v", err)
-	}
-
 	logger.Info("packing dimg")
 	dimgPath := filepath.Join(outputPath, "image.dimg")
-	err = di3fsImage.PackDimg(logger, tempDiffDir, entry, "", dimgPath)
+	err = di3fsImage.PackDir(tempDir, dimgPath)
 	if err != nil {
 		return fmt.Errorf("failed to pack dimg: %v", err)
 	}

cmd/diff/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"time"
@@ -18,7 +17,6 @@ var logger = log.G(context.TODO())
 func main() {
 	logger.Logger.SetLevel(logrus.WarnLevel)
 	if len(os.Args) < 6 {
-		fmt.Println("diff base-dir new-dir output-dir json-file mode [benchmark]")
 		fmt.Println("diff dimg base-dimg new-dimg output-dimg mode [benchmark]")
 		os.Exit(1)
 	}
@@ -57,37 +55,8 @@ func main() {
 			panic(err)
 		}
 	} else {
-		baseDir := os.Args[1]
-		newDir := os.Args[2]
-		outputDir := os.Args[3]
-		jsonPath := os.Args[4]
-		mode = os.Args[5]
-		if mode != "binary-diff" && mode != "file-diff" {
-			fmt.Println("mode is \"binary-diff\" or \"file-diff\"")
-			os.Exit(1)
-		}
-		base = baseDir
-		new = newDir
-
-		os.RemoveAll(outputDir)
-		os.RemoveAll(jsonPath)
-
-		entry, err := image.GenerateDiffFromDir(baseDir, newDir, outputDir, mode == "binary-diff", baseDir != "")
-		if err != nil {
-			panic(err)
-		}
-
-		//entry.print("", true)
-		entryJson, err := json.MarshalIndent(entry, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-		jsonFile, err := os.Create(jsonPath)
-		if err != nil {
-			panic(err)
-		}
-		defer jsonFile.Close()
-		jsonFile.Write(entryJson)
+		fmt.Println("only 'dimg' mode is allowed")
+		os.Exit(1)
 	}
 
 	if benchmarkEnabled {

cmd/merge/main.go

@@ -1,21 +1,18 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"time"
 
-	"github.com/jinzhu/copier"
 	"github.com/naoki9911/fuse-diff-containerd/pkg/benchmark"
-	"github.com/naoki9911/fuse-diff-containerd/pkg/di3fs"
+	"github.com/naoki9911/fuse-diff-containerd/pkg/image"
 	log "github.com/sirupsen/logrus"
 )
 
 func main() {
 	log.SetLevel(log.WarnLevel)
 	if len(os.Args) < 5 {
-		fmt.Println("merge dir lower-diff lower-json upper-diff upper-json merged-diff merged-json")
 		fmt.Println("merge dimg lower-dimg upper-dimg merged-dimg benchmark")
 		os.Exit(1)
 	}
@@ -41,7 +38,7 @@ func main() {
 			panic(err)
 		}
 		defer mergeFile.Close()
-		err = di3fs.MergeDimg(lowerDimg, upperDimg, mergeFile)
+		err = image.MergeDimg(lowerDimg, upperDimg, mergeFile)
 		if err != nil {
 			panic(err)
 		}
@@ -62,53 +59,7 @@ func main() {
 			}
 		}
 	} else {
-		lowerDiff := os.Args[2]
-		lowerJson := os.Args[3]
-		upperDiff := os.Args[4]
-		upperJson := os.Args[5]
-		mergedDiff := os.Args[6]
-		mergedJson := os.Args[7]
-
-		lowerJsonRaw, err := os.ReadFile(lowerJson)
-		if err != nil {
-			panic(err)
-		}
-		lowerEntry := di3fs.FileEntry{}
-		err = json.Unmarshal(lowerJsonRaw, &lowerEntry)
-		if err != nil {
-			panic(err)
-		}
-
-		upperJsonRaw, err := os.ReadFile(upperJson)
-		if err != nil {
-			panic(err)
-		}
-		upperEntry := di3fs.FileEntry{}
-		err = json.Unmarshal(upperJsonRaw, &upperEntry)
-		if err != nil {
-			panic(err)
-		}
-		mergedEntry := di3fs.NewFileEntry()
-		err = copier.Copy(mergedEntry, upperEntry)
-		if err != nil {
-			panic(err)
-		}
-
-		err = di3fs.MergeDiff(lowerDiff, upperDiff, mergedDiff, &lowerEntry, &upperEntry, mergedEntry)
-		if err != nil {
-			panic(err)
-		}
-
-		//entry.print("", true)
-		entryJson, err := json.MarshalIndent(mergedEntry, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-		jsonFile, err := os.Create(mergedJson)
-		if err != nil {
-			panic(err)
-		}
-		defer jsonFile.Close()
-		jsonFile.Write(entryJson)
+		fmt.Println("only 'dimg' mode is allowed")
+		os.Exit(1)
 	}
 }

cmd/patch/main.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -65,12 +64,10 @@ func applyFilePatchForGz(baseFilePath, newFilePath string, patch io.Reader) erro
 	return nil
 }
 
-func applyPatch(basePath, newPath, patchPath string, dirEntry di3fs.FileEntry, image *os.File, imageStartOffset int64, isBase bool) error {
+func applyPatch(basePath, newPath string, dirEntry *di3fs.FileEntry, image *os.File, imageStartOffset int64, isBase bool) error {
 	fName := dirEntry.Name
-	isImage := image != nil
 	baseFilePath := path.Join(basePath, fName)
 	newFilePath := path.Join(newPath, fName)
-	patchFilePath := path.Join(patchPath, dirEntry.DiffName)
 
 	if isBase && !dirEntry.IsDir() && !dirEntry.IsSymlink() && !dirEntry.IsNew() {
 		return fmt.Errorf("invalid base image %q", newFilePath)
@@ -97,13 +94,12 @@ func applyPatch(basePath, newPath, patchPath string, dirEntry di3fs.FileEntry, i
 			return err
 		}
 	} else if dirEntry.IsDir() {
-		patchFilePath = path.Join(patchPath, dirEntry.Name)
 		err := os.Mkdir(newFilePath, os.ModePerm)
 		if err != nil {
 			return err
 		}
 		for _, c := range dirEntry.Childs {
-			err = applyPatch(baseFilePath, newFilePath, patchFilePath, c, image, imageStartOffset, isBase)
+			err = applyPatch(baseFilePath, newFilePath, c, image, imageStartOffset, isBase)
 			if err != nil {
 				return err
 			}
@@ -117,51 +113,35 @@ func applyPatch(basePath, newPath, patchPath string, dirEntry di3fs.FileEntry, i
 		//if strings.Contains(dirEntry.Name, ".wh") {
 		//	fmt.Println(newFilePath)
 		//}
-		if isImage {
-			logger.Debugf("copy %q from image(offset=%d size=%d)", newFilePath, dirEntry.Offset, dirEntry.CompressedSize)
-			patchBytes := make([]byte, dirEntry.CompressedSize)
-			_, err := image.ReadAt(patchBytes, dirEntry.Offset+imageStartOffset)
-			if err != nil {
-				return err
-			}
-			patchBuf := bytes.NewBuffer(patchBytes)
-			patchReader, err := zstd.NewReader(patchBuf)
-			if err != nil {
-				return err
-			}
-			defer patchReader.Close()
-
-			newFile, err := os.Create(newFilePath)
-			if err != nil {
-				return err
-			}
-			defer newFile.Close()
+		logger.Debugf("copy %q from image(offset=%d size=%d)", newFilePath, dirEntry.Offset, dirEntry.CompressedSize)
+		patchBytes := make([]byte, dirEntry.CompressedSize)
+		_, err := image.ReadAt(patchBytes, dirEntry.Offset+imageStartOffset)
+		if err != nil {
+			return err
+		}
+		patchBuf := bytes.NewBuffer(patchBytes)
+		patchReader, err := zstd.NewReader(patchBuf)
+		if err != nil {
+			return err
+		}
+		defer patchReader.Close()
 
-			io.Copy(newFile, patchReader)
-		} else {
-			err := cp.Copy(patchFilePath, newFilePath)
-			if err != nil {
-				return err
-			}
+		newFile, err := os.Create(newFilePath)
+		if err != nil {
+			return err
 		}
+		defer newFile.Close()
+
+		io.Copy(newFile, patchReader)
 	} else if dirEntry.Type == di3fs.FILE_ENTRY_FILE_DIFF {
 		var patchReader io.Reader
-		if isImage {
-			logger.Debugf("applying diff to %q from image(offset=%d size=%d)", newFilePath, dirEntry.Offset, dirEntry.CompressedSize)
-			patchBytes := make([]byte, dirEntry.CompressedSize)
-			_, err := image.ReadAt(patchBytes, dirEntry.Offset+imageStartOffset)
-			if err != nil {
-				return err
-			}
-			patchReader = bytes.NewBuffer(patchBytes)
-		} else {
-			patchFile, err := os.Open(patchFilePath)
-			if err != nil {
-				return err
-			}
-			defer patchFile.Close()
-			patchReader = patchFile
+		logger.Debugf("applying diff to %q from image(offset=%d size=%d)", newFilePath, dirEntry.Offset, dirEntry.CompressedSize)
+		patchBytes := make([]byte, dirEntry.CompressedSize)
+		_, err := image.ReadAt(patchBytes, dirEntry.Offset+imageStartOffset)
+		if err != nil {
+			return err
 		}
+		patchReader = bytes.NewBuffer(patchBytes)
 		if dirEntry.UncompressedGz {
 			err := applyFilePatchForGz(baseFilePath, newFilePath, patchReader)
 			if err != nil {
@@ -194,14 +174,17 @@ func applyPatch(basePath, newPath, patchPath string, dirEntry di3fs.FileEntry, i
 func main() {
 	logger.Logger.SetLevel(logrus.WarnLevel)
 	if len(os.Args) < 5 {
-		fmt.Println("diff dir base-dir new-dir [patch-dir|patch-img] json-file")
-		fmt.Println("diff dimg base-dir new-dir patch-img [benchmark]")
+		fmt.Println("diff dimg base-dir new-dir patch-dimg [benchmark]")
 		os.Exit(1)
 	}
+
 	mode := os.Args[1]
+	if mode != "dimg" {
+		logger.Fatal("only dimg mode is allowed")
+	}
 	baseDir := os.Args[2]
 	newDir := os.Args[3]
-	patchDir := os.Args[4]
+	patchDimg := os.Args[4]
 
 	os.RemoveAll(newDir)
 
@@ -217,11 +200,11 @@ func main() {
 			defer b.Close()
 		}
 		start := time.Now()
-		imageHeader, imageFile, curOffset, err := di3fs.LoadImage(patchDir)
+		imageHeader, imageFile, curOffset, err := di3fs.LoadImage(patchDimg)
 		if err != nil {
 			panic(err)
 		}
-		err = applyPatch(baseDir, newDir, patchDir, imageHeader.FileEntry, imageFile, curOffset, imageHeader.BaseId == "")
+		err = applyPatch(baseDir, newDir, &imageHeader.FileEntry, imageFile, curOffset, imageHeader.BaseId == "")
 		if err != nil {
 			panic(err)
 		}
@@ -240,20 +223,5 @@ func main() {
 				panic(err)
 			}
 		}
-	} else {
-		entry := di3fs.FileEntry{}
-		jsonPath := os.Args[5]
-		jsonRaw, err := os.ReadFile(jsonPath)
-		if err != nil {
-			panic(err)
-		}
-		err = json.Unmarshal(jsonRaw, &entry)
-		if err != nil {
-			panic(err)
-		}
-		err = applyPatch(baseDir, newDir, patchDir, entry, nil, 0, false)
-		if err != nil {
-			panic(err)
-		}
 	}
 }