handle multiple encodings

Signed-off-by: Naoki MATSUMOTO <[email protected]>

.github/workflows/test.yaml

@@ -105,8 +105,9 @@ jobs:
       matrix:
         script: ["apache", "mysql", "nginx", "postgres", "redis"]
         thread: ["4"]
-        sched: ["none", "size-ordered"]
+        sched: ["none"]
         comp: ["bzip2", "zstd"]
+        encoding: ["bsdiffx", "xdelta3"]
     steps:
       - uses: actions/checkout@v4
       - name: setup lxd
@@ -122,7 +123,7 @@ jobs:
       - name: launch lxc container
         run: ./tests/launch_test_lxc.sh test-export
       - name: run benchmark (${{ matrix.script }})
-        run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && RUNNER=GHA sudo -E /home/ubuntu/d4c/tests/bench_single.sh benchmark benchmark/images ${{ matrix.script }} ${{ matrix.thread }} ${{ matrix.sched }} ${{ matrix.comp }}"
+        run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && RUNNER=GHA sudo -E /home/ubuntu/d4c/tests/bench_single.sh benchmark benchmark/images ${{ matrix.script }} ${{ matrix.thread }} ${{ matrix.sched }} ${{ matrix.comp }} ${{ matrix.encoding }}"
 
   benchmark-cdimg:
     runs-on: ubuntu-22.04

Makefile

@@ -20,7 +20,7 @@ fuse-diff:
 	go build -o fuse-diff ./cmd/fuse-diff
 
 plugins:
-	go build -buildmode=plugin -o plugin_basic.so ./cmd/plugins/basic/main.go
+	go build -buildmode=plugin -o plugin_bsdiffx.so ./cmd/plugins/bsdiffx/main.go
 	go build -buildmode=plugin -o plugin_xdelta3.so ./cmd/plugins/xdelta3/main.go
 
 clean:

cmd/ctr-cli/diff/diff.go

@@ -78,6 +78,12 @@ func DimgCommand() *cli.Command {
 				Value:    "bzip2",
 				Required: false,
 			},
+			&cli.StringFlag{
+				Name:     "deltaEncoding",
+				Usage:    "algorithm for delta encoding (bsdiffx, xdelta3, mixted)",
+				Value:    "bsdiffx",
+				Required: false,
+			},
 		},
 	}
 
@@ -133,6 +139,7 @@ func dimgAction(c *cli.Context) error {
 		CompressionMode:  compMode,
 		BenchmarkPerFile: enableBenchPerFile,
 		Benchmarker:      b,
+		DeltaEncoding:    c.String("deltaEncoding"),
 	}
 	err = image.GenerateDiffFromDimg(oldDimg, newDimg, outDimg, mode == ModeDiffBinary, dc, pm)
 	if err != nil {
@@ -224,6 +231,12 @@ func CdimgCommand() *cli.Command {
 				Value:    "bzip2",
 				Required: false,
 			},
+			&cli.StringFlag{
+				Name:     "deltaEncoding",
+				Usage:    "algorithm for delta encoding (bsdiffx, xdelta3, mixted)",
+				Value:    "bsdiffx",
+				Required: false,
+			},
 		},
 	}
 
@@ -279,6 +292,7 @@ func cdimgAction(c *cli.Context) error {
 		CompressionMode:  compMode,
 		BenchmarkPerFile: enableBenchPerFile,
 		Benchmarker:      b,
+		DeltaEncoding:    c.String("deltaEncoding"),
 	}
 	err = image.GenerateDiffFromCdimg(oldCdimg, newCdimg, outCdimg, mode == ModeDiffBinary, dc, pm)
 	if err != nil {

cmd/ctr-cli/stat/stat.go

@@ -1,6 +1,7 @@
 package stat
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -257,8 +258,7 @@ func diffImpl(pathA, pathB string, pm *bsdiffx.PluginManager, b *benchmark.Bench
 			return fmt.Errorf("failed to open and readall %s: %v", pathB, err)
 		}
 
-		p := pm.GetPluginByExt(filepath.Ext(pathA))
-		if !p.Compare(fileA, fileB) {
+		if !bytes.Equal(fileA, fileB) {
 			return fmt.Errorf("unmatched content %s and %s", fileA, fileB)
 		}
 

cmd/plugins/basic/main.go → cmd/plugins/bsdiffx/main.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 
+	"github.com/google/uuid"
 	"github.com/naoki9911/fuse-diff-containerd/pkg/bsdiffx"
 )
 
@@ -27,5 +28,9 @@ func Compare(a, b []byte) bool {
 	return bytes.Equal(a, b)
 }
 
+func ID() uuid.UUID {
+	return uuid.MustParse("2466ff1d-27f2-4cc5-a6ef-bf7de2b7a05f")
+}
+
 func init() {
 }

cmd/plugins/xdelta3/main.go

@@ -160,5 +160,9 @@ func Compare(a, b []byte) bool {
 	return bytes.Equal(a, b)
 }
 
+func ID() uuid.UUID {
+	return uuid.MustParse("c4e21629-5937-49d2-9646-b93df8e04b5d")
+}
+
 func init() {
 }

pkg/bsdiffx/plugin.go

@@ -17,6 +17,7 @@ type PluginEntry struct {
 	Uuid uuid.UUID `json:"uuid"`
 	Path string    `json:"path"`
 	Ext  string    `json:"ext"`
+	Size int       `json:"size"`
 
 	p *Plugin `json:"-"`
 }
@@ -31,21 +32,20 @@ func LoadOrDefaultPlugins(path string) (*PluginManager, error) {
 	if err != nil {
 		return nil, err
 	}
-	basicPlugin := filepath.Join(filepath.Dir(d4cBinPath), "plugin_basic.so")
 	xdelta3Plugin := filepath.Join(filepath.Dir(d4cBinPath), "plugin_xdelta3.so")
+	bsdiffxPlugin := filepath.Join(filepath.Dir(d4cBinPath), "plugin_bsdiffx.so")
 	_ = xdelta3Plugin
 	mgr := &PluginManager{
 		defaultPlugin: DefaultPluigin(),
 		plugins: []PluginEntry{
 			{
 				Name: "xdelta3",
 				Path: xdelta3Plugin,
-				Ext:  "",
+				Size: 512 * 1024 * 1024, // handle more than 512 MiB file
 			},
 			{
-				Name: "basic",
-				Path: basicPlugin,
-				Ext:  ".xz",
+				Name: "bsdiffx",
+				Path: bsdiffxPlugin,
 			},
 		},
 	}
@@ -57,6 +57,7 @@ func LoadOrDefaultPlugins(path string) (*PluginManager, error) {
 				return nil, fmt.Errorf("failed to open %s (%s): %v", pe.Name, pe.Path, err)
 			}
 			pe.p = p
+			pe.Uuid = p.ID()
 		}
 		return mgr, nil
 	}
@@ -82,24 +83,55 @@ func LoadOrDefaultPlugins(path string) (*PluginManager, error) {
 func (pm *PluginManager) GetPluginByExt(ext string) *Plugin {
 	for i := range pm.plugins {
 		pe := pm.plugins[i]
-		if pe.Ext == "" {
+		if pe.Ext == ext {
 			return pe.p
 		}
-		if pe.Ext == ext {
+	}
+
+	return pm.defaultPlugin
+}
+
+func (pm *PluginManager) GetPluginByName(name string) *Plugin {
+	for i := range pm.plugins {
+		pe := pm.plugins[i]
+		if pe.Name == name {
+			return pe.p
+		}
+	}
+
+	return nil
+}
+
+func (pm *PluginManager) GetPluginBySize(size int) *Plugin {
+	for i := range pm.plugins {
+		pe := pm.plugins[i]
+		if size > pe.Size {
 			return pe.p
 		}
 	}
 
 	return pm.defaultPlugin
 }
 
+func (pm *PluginManager) GetPluginByUuid(uuid uuid.UUID) *Plugin {
+	for i := range pm.plugins {
+		pe := pm.plugins[i]
+		if pe.Uuid == uuid {
+			return pe.p
+		}
+	}
+
+	return nil
+}
+
 type Plugin struct {
 	p       *plugin.Plugin
 	info    func() string
 	diff    func(oldBytes, newBytes []byte, patchWriter io.Writer, mode CompressionMode) error
 	patch   func(oldBytes []byte, patchReader io.Reader) ([]byte, error)
 	merge   func(lowerDiff, upperDiff io.Reader, mergedDiff io.Writer) error
 	compare func(a, b []byte) bool
+	id      func() uuid.UUID
 }
 
 func defaultPluginInfo() string {
@@ -163,6 +195,12 @@ func OpenPlugin(path string) (*Plugin, error) {
 	}
 	plugin.compare = sCompare.(func(a, b []byte) bool)
 
+	sID, err := p.Lookup("ID")
+	if err != nil {
+		return nil, err
+	}
+	plugin.id = sID.(func() uuid.UUID)
+
 	return plugin, nil
 }
 
@@ -185,3 +223,7 @@ func (p *Plugin) Merge(lowerDiff, upperDiff io.Reader, mergedDiff io.Writer) err
 func (p *Plugin) Compare(a, b []byte) bool {
 	return p.compare(a, b)
 }
+
+func (p *Plugin) ID() uuid.UUID {
+	return p.id()
+}

pkg/di3fs/di3fs.go

@@ -348,8 +348,11 @@ func (dr *Di3fsRoot) IsBase() bool {
 
 func newNode(fe *image.FileEntry, baseFE []*image.FileEntry, root *Di3fsRoot) *Di3fsNode {
 	var p *bsdiffx.Plugin = nil
-	if root != nil {
-		p = root.pm.GetPluginByExt(filepath.Ext(fe.Name))
+	if root != nil && fe.Type == image.FILE_ENTRY_FILE_DIFF {
+		p = root.pm.GetPluginByUuid(fe.PluginUuid)
+		if p == nil {
+			panic(fmt.Sprintf("plugin for %s not found", fe.PluginUuid))
+		}
 	}
 	node := &Di3fsNode{
 		openCount: 0,

pkg/image/diff.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 	"sort"
 	"strconv"
 	"sync"
@@ -164,6 +163,7 @@ type DiffConfig struct {
 	CompressionMode  bsdiffx.CompressionMode
 	BenchmarkPerFile bool
 	Benchmarker      *benchmark.Benchmark
+	DeltaEncoding    string
 }
 
 func (dc *DiffConfig) Validate() error {
@@ -366,7 +366,16 @@ func generateDiffMultithread(oldDimgFile, newDimgFile *DimgFile, oldEntry, newEn
 						continue
 					}
 					if len(oldBytes) > 0 && isBinaryDiff {
-						p := pm.GetPluginByExt(filepath.Ext(dt.newEntry.Name))
+						var p *bsdiffx.Plugin = nil
+						switch dc.DeltaEncoding {
+						case "mixed":
+							p = pm.GetPluginBySize(dt.newEntry.Size)
+						default:
+							p = pm.GetPluginByName(dc.DeltaEncoding)
+						}
+						if p == nil {
+							panic(fmt.Sprintf("unknown delta encoding %s", dc.DeltaEncoding))
+						}
 						// old File may be 0-bytes
 						diffWriter := new(bytes.Buffer)
 						//fmt.Printf("oldBytes=%d newBytes=%d old=%v new=%v\n", len(oldBytes), len(newBytes), *oldChildEntry, *newChildEntry)
@@ -378,6 +387,7 @@ func generateDiffMultithread(oldDimgFile, newDimgFile *DimgFile, oldEntry, newEn
 						dt.newEntry.Type = FILE_ENTRY_FILE_DIFF
 						dt.newEntry.CompressedSize = int64(diffWriter.Len())
 						dt.data = diffWriter.Bytes()
+						dt.newEntry.PluginUuid = p.ID()
 					} else {
 						dt.newEntry.Type = FILE_ENTRY_FILE_NEW
 						dt.data = make([]byte, dt.newEntry.CompressedSize)

pkg/image/fs.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
 )
@@ -82,6 +83,7 @@ type FileEntry struct {
 	CompressedSize int64         `json:"compressedSize,omitempty"`
 	Offset         int64         `json:"offset,omitempty"`
 	Digest         digest.Digest `json:"digest"`
+	PluginUuid     uuid.UUID     `json:"pluginUuid"`
 }
 
 func (fe *FileEntry) DeepCopy() *FileEntry {

pkg/image/merge.go

@@ -100,7 +100,7 @@ func mergeDiffDimgMultihread(lowerImgFile, upperImgFile *DimgFile, mergeOut *byt
 					start := time.Now()
 					mode := ""
 					if mt.lowerEntry != nil && mt.upperEntry != nil {
-						p := pm.GetPluginByExt(filepath.Ext(mt.upperEntry.Name))
+						p := pm.GetPluginByUuid(mt.upperEntry.PluginUuid)
 						if mt.lowerEntry.Type == FILE_ENTRY_FILE_NEW && mt.upperEntry.Type == FILE_ENTRY_FILE_DIFF {
 							lowerBytes := make([]byte, mt.lowerEntry.CompressedSize)
 							upperBytes := make([]byte, mt.upperEntry.CompressedSize)
@@ -146,6 +146,9 @@ func mergeDiffDimgMultihread(lowerImgFile, upperImgFile *DimgFile, mergeOut *byt
 							mt.data = mergeCompressed
 							mode = "apply"
 						} else if mt.lowerEntry.Type == FILE_ENTRY_FILE_DIFF && mt.upperEntry.Type == FILE_ENTRY_FILE_DIFF {
+							if mt.lowerEntry.PluginUuid != mt.upperEntry.PluginUuid {
+								panic(fmt.Sprintf("unmatched plugin lower %s upper %s", mt.lowerEntry.PluginUuid, mt.upperEntry.PluginUuid))
+							}
 							lowerBytes := make([]byte, mt.lowerEntry.CompressedSize)
 							upperBytes := make([]byte, mt.upperEntry.CompressedSize)
 							_, err := lowerImgFile.ReadAt(lowerBytes, mt.lowerEntry.Offset)

pkg/image/patch.go

@@ -167,7 +167,7 @@ func applyPatchImpl(basePath, newPath string, dirEntry *FileEntry, img *DimgFile
 			return nil, err
 		}
 	} else if dirEntry.Type == FILE_ENTRY_FILE_DIFF {
-		p := pm.GetPluginByExt(filepath.Ext(dirEntry.Name))
+		p := pm.GetPluginByUuid(dirEntry.PluginUuid)
 		var patchReader io.Reader
 		logger.Debugf("applying diff to %q from image(offset=%d size=%d)", newFilePath, dirEntry.Offset, dirEntry.CompressedSize)
 		patchBytes := make([]byte, dirEntry.CompressedSize)

tests/bench.sh

@@ -19,24 +19,21 @@ IMAGE_DIR=$RESULT_DIR/images
 mkdir -p $IMAGE_DIR
 mkdir -p /tmp/benchmark
 
-#THREADS=("1" "8")
 TESTS=("apache" "mysql" "nginx" "postgres" "redis")
-#SCHED_MODES=("none" "size-ordered")
+THREADS=("1" "8")
 SCHED_MODES=("none")
-#COMP_MODES=("bzip2" "zstd")
 COMP_MODES=("bzip2")
+ENCODINGS=("bsdiffx" "xdelta3")
 for TEST in "${TESTS[@]}"; do
-	for COMP_MODE in "${COMP_MODES[@]}"; do
-			./bench_single.sh $RESULT_DIR $IMAGE_DIR $TEST 1 size-ordered $COMP_MODE
-	done
-	for SCHED_MODE in "${SCHED_MODES[@]}"; do
-		for COMP_MODE in "${COMP_MODES[@]}"; do
-			./bench_single.sh $RESULT_DIR $IMAGE_DIR $TEST 8 $SCHED_MODE $COMP_MODE
+	for THREAD in "${THREADS[@]}"; do
+		for ENCODING in "${ENCODINGS[@]}"; do
+			./bench_single.sh $RESULT_DIR $IMAGE_DIR $TEST $THREAD none bzip2 $ENCODING
+
+			cat $RESULT_DIR/$TEST-benchmark.log >> /tmp/benchmark/benchmark.log
+			cat $RESULT_DIR/$TEST-benchmark-io.log >> /tmp/benchmark/benchmark-io.log
+			cat $RESULT_DIR/$TEST-compare.log >> /tmp/benchmark/compare.log
 		done
 	done
-	cat $RESULT_DIR/$TEST-benchmark.log >> /tmp/benchmark/benchmark.log
-	cat $RESULT_DIR/$TEST-benchmark-io.log >> /tmp/benchmark/benchmark-io.log
-	cat $RESULT_DIR/$TEST-compare.log >> /tmp/benchmark/compare.log
 done
 
 python3 ./plot_diff.py /tmp/benchmark/benchmark.log /tmp/benchmark/diff.png