tmp

.github/workflows/test.yaml

@@ -0,0 +1,132 @@
+---
+name: Test
+on:
+  push:
+    branches:
+      - master
+      - main
+      - release/**
+      - gha
+  pull_request: null
+  workflow_dispatch:
+
+jobs:
+  golangci-lint:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.x
+      - run: sudo apt-get update
+      - name: golangci-lint
+        uses: golangci/[email protected]
+        with:
+          version: v1.55.2
+          args: --verbose
+
+  create-lxc-image:
+    name: create-lxc-image
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+         key: lxc-image-base-${{ hashFiles('go.sum', 'tests/setup_test.sh') }}
+         path: /tmp/test-image.tar.zst
+         lookup-only: true
+
+      - name: setup lxd
+        id: s1
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: ./tests/setup_lxd.sh
+
+      - name: launch lxc container
+        id: s2
+        if: steps.s1.conclusion == 'success'
+        run: ./tests/launch_test_lxc.sh
+
+      - name: install dependencies and build
+        id: s3
+        if: steps.s2.conclusion == 'success'
+        run: sudo lxc exec test -- sudo --login --user ubuntu /host/tests/setup_test.sh
+
+      - name: export image
+        id: s4
+        if: steps.s3.conclusion == 'success'
+        run: ./tests/export_lxc_image.sh test
+
+      - uses: actions/cache/save@v3
+        id: s5
+        if: steps.s4.conclusion == 'success'
+        with:
+          key: lxc-image-base-${{ hashFiles('go.sum', 'tests/setup_test.sh') }}
+          path: /tmp/test-image.tar.zst
+
+  sync:
+    runs-on: ubuntu-22.04
+    needs: create-lxc-image
+    timeout-minutes: 20
+    steps:
+      - uses: actions/[email protected]
+      - name: setup lxd
+        run: ./tests/setup_lxd.sh
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+         key: lxc-image-base-${{ hashFiles('go.sum', 'tests/setup_test.sh') }}
+         path: /tmp/test-image.tar.zst
+         fail-on-cache-miss: true
+      - name: load lxc image
+        run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export
+      - name: launch lxc container
+        run: ./tests/launch_test_lxc.sh test-export
+      - name: run test
+        run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/d4c/tests/sync_lxc.sh"
+      # some source codes may be updated. re-export new image.
+      - name: export image
+        run: sudo lxc image alias delete test-export && rm -f /tmp/test-image.tar.zst && ./tests/export_lxc_image.sh test
+      - uses: actions/cache/save@v3
+        with:
+          key: lxc-image-${{ github.sha }}
+          path: /tmp/test-image.tar.zst
+
+  benchmark:
+    runs-on: ubuntu-22.04
+    needs: sync
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        script: ["apache", "mysql", "nginx", "postgres", "redis"]
+    steps:
+      - uses: actions/[email protected]
+      - name: setup lxd
+        run: ./tests/setup_lxd.sh
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+         key: lxc-image-${{ github.sha }}
+         path: /tmp/test-image.tar.zst
+         fail-on-cache-miss: true
+      - name: load lxc image
+        run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export
+      - name: launch lxc container
+        run: ./test/launch_test_lxc.sh test-export
+      - name: run benchmark (${{ matrix.script }})
+        run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/d4c/tests/bench_lxc.sh ${{ matrix.script }}"
+      - name: get result
+        id: get_result
+        run: |
+          mkdir /tmp/benchmark-results
+          sudo lxc file pull test/home/ubuntu/d4c/tests/benchmark/${{ matrix.script }}-benchmark.log /tmp/benchmark-results/.
+      - uses: actions/upload-artifact@v3
+        if: steps.get_result.conclusion == 'success'
+        with:
+          name: benchmark-${{ matrix.script }}
+          path: /tmp/benchmark-results

.gitignore

@@ -13,3 +13,6 @@
 /diff
 /pack
 /server
+/fuse-diff
+/merge
+/patch

Makefile

@@ -1,6 +1,6 @@
-.PHONY: snapshotter ctr-cli diff-tools server
+.PHONY: snapshotter ctr-cli diff-tools server fuse-diff merge patch
 
-all: snapshotter ctr-cli diff-tools server
+all: snapshotter ctr-cli diff-tools server fuse-diff merge patch
 
 run:
 	make clean
@@ -20,5 +20,14 @@ diff-tools:
 server:
 	go build -o server ./cmd/server
 
+fuse-diff:
+	go build -o fuse-diff ./cmd/fuse-diff
+
+merge:
+	go build -o merge ./cmd/merge
+
+patch:
+	go build -o patch ./cmd/patch
+
 clean:
-	rm -f snapshotter ctr-cli diff pack server
+	rm -f snapshotter ctr-cli diff pack server fuse-diff merge patch

tests/.gitignore

@@ -1,11 +1 @@
-*
-!/.gitignore
-!/postgres
-!/simple-dir
-!/run-tests.sh
-!/bench*.sh
-!/bench-*.py
-!/push-*.sh
-!/scp*.sh
-!version.sh
-/server
+/benchmark_*

tests/bench.sh

@@ -7,19 +7,23 @@ if [ $EUID -ne 0 ]; then
 fi
 RUN_NUM=1
 
+PATH=$PATH:/usr/local/go/bin
+
+cd $(cd $(dirname $0); pwd)
+pushd ../
+make all
+popd
+
 RESULT_DIR=benchmark_`date +%Y-%m-%d-%H%M`
-mkdir -p $RESULT_DIR
+IMAGE_DIR=$RESULT_DIR/images
+mkdir -p $IMAGE_DIR
 
-IMAGES=("apach-root" "mysql-root" "nginx-root" "postgres-root" "redis-root")
-for IMAGE in "${IMAGES[@]}"
+TESTS=("apache" "mysql" "nginx" "postgres" "redis")
+for TEST in "${TESTS[@]}"
 do
-	echo "Benchmarking $IMAGE"
-	cd $IMAGE
-	rm -f benchmark.log
-	ln -sf ../bench_impl.sh ./bench.sh
-	./bench.sh $RUN_NUM
-	cp benchmark.log ../$RESULT_DIR/$IMAGE-benchmark.log
-	cd ../
+	echo "Benchmarking $TEST"
+	./bench_impl.sh test_$TEST.sh $IMAGE_DIR $RUN_NUM
+	cp $IMAGE_DIR/$TEST/benchmark.log ./$RESULT_DIR/$TEST-benchmark.log
 done
 
 cd $RESULT_DIR

tests/bench_impl.sh

@@ -3,8 +3,19 @@
 set -eu
 
 
-source ./version.sh
-RUN_NUM=$1
+ROOT_DIR=$(cd $(dirname $0)/../; pwd)
+BIN_CTR_CLI="$ROOT_DIR/ctr-cli"
+BIN_DIFF="$ROOT_DIR/diff"
+BIN_PACK="$ROOT_DIR/pack"
+BIN_PATCH="$ROOT_DIR/patch"
+BIN_FUSE="$ROOT_DIR/fuse-diff"
+BIN_MERGE="$ROOT_DIR/merge"
+
+TEST_SCRIPT=$1
+IMAGE_DIR=$2
+RUN_NUM=$3
+
+source $TEST_SCRIPT
 
 mkdir -p /tmp/fuse
 
@@ -15,24 +26,26 @@ function err() {
 
 trap err ERR
 
-
-rm -f diff patch pack fuse-diff merge
-go build ../../cmd/diff
-go build ../../cmd/patch
-go build ../../cmd/pack
-go build ../../cmd/fuse-diff
-go build ../../cmd/merge
+IMAGE_DIR=$IMAGE_DIR/$IMAGE_NAME
+mkdir -p $IMAGE_DIR
+cd $IMAGE_DIR
 
 for ((i=0; i < ${#IMAGE_VERSIONS[@]}; i++));do
 	IMAGE=${IMAGE_VERSIONS[i]}
-	echo "Creating base image for $IMAGE"
-	./diff "" $IMAGE $IMAGE-base $IMAGE-base.json binary-diff
-	./pack $IMAGE-base $IMAGE-base.json "" $IMAGE-base.dimg
-	./patch dimg "" $IMAGE-base-patched $IMAGE-base.dimg
+	echo "Creating base image for $IMAGE_NAME:$IMAGE"
+	$BIN_CTR_CLI convert --image $DOCKER_IMAGE:$IMAGE --output ./image-$IMAGE >/dev/null 2>&1
+	mkdir $IMAGE
+	sudo tar -xf ./image-$IMAGE/layer.tar -C ./$IMAGE
+
+	# remove '/dev' dir
+	sudo rm -rf ./$IMAGE/dev
+
+	$BIN_DIFF "" $IMAGE $IMAGE-base $IMAGE-base.json binary-diff
+	$BIN_PACK $IMAGE-base $IMAGE-base.json "" $IMAGE-base.dimg
+	$BIN_PATCH dimg "" $IMAGE-base-patched $IMAGE-base.dimg
 	diff -r $IMAGE $IMAGE-base-patched --no-dereference
 done
 
-
 for ((i=0; i < $(expr ${#IMAGE_VERSIONS[@]} - 1); i++));do
 	LOWER=${IMAGE_VERSIONS[i]}
 	UPPER=${IMAGE_VERSIONS[$(expr $i + 1)]}
@@ -42,26 +55,26 @@ for ((i=0; i < $(expr ${#IMAGE_VERSIONS[@]} - 1); i++));do
 	for ((j=0; j < $RUN_NUM; j++));do
 		NOW_COUNT=$(expr $j + 1)
 		echo "Benchmark diff $DIFF_NAME binary-diff ($NOW_COUNT/$RUN_NUM)"
-		./diff $LOWER $UPPER diff_$DIFF_NAME diff_$DIFF_NAME.json binary-diff benchmark
+		$BIN_DIFF $LOWER $UPPER diff_$DIFF_NAME diff_$DIFF_NAME.json binary-diff benchmark
 	done
 
 	# packing diff data
-	./pack diff_$DIFF_NAME diff_$DIFF_NAME.json $LOWER-base.dimg diff_$DIFF_NAME.dimg
+	$BIN_PACK diff_$DIFF_NAME diff_$DIFF_NAME.json $LOWER-base.dimg diff_$DIFF_NAME.dimg
 	ls -l diff_$DIFF_NAME.dimg
 
 	# patching diff data
 	for ((j=0; j < $RUN_NUM; j++));do
 		NOW_COUNT=$(expr $j + 1)
 		echo "Benchmark patch $DIFF_NAME binary-diff ($NOW_COUNT/$RUN_NUM)"
-		./patch dimg $LOWER $UPPER-patched diff_$DIFF_NAME.dimg benchmark
+		$BIN_PATCH dimg $LOWER $UPPER-patched diff_$DIFF_NAME.dimg benchmark
 	done
 	diff -r $UPPER $UPPER-patched --no-dereference
 
 	# mount with di3fs
 	for ((j=0; j < $RUN_NUM; j++));do
 		NOW_COUNT=$(expr $j + 1)
 		echo "Benchmark di3fs $DIFF_NAME binary-diff ($NOW_COUNT/$RUN_NUM)"
-		./fuse-diff --basedir=./$LOWER-base.dimg --patchdir=./diff_$DIFF_NAME.dimg --mode=dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 &
+		$BIN_FUSE --basedir=./$LOWER-base.dimg --patchdir=./diff_$DIFF_NAME.dimg --mode=dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 &
 		sleep 1
 		if [ $j -eq 0 ]; then
 			diff -r $UPPER /tmp/fuse --no-dereference
@@ -73,14 +86,14 @@ for ((i=0; i < $(expr ${#IMAGE_VERSIONS[@]} - 1); i++));do
 	for ((j=0; j < $RUN_NUM; j++));do
 		NOW_COUNT=$(expr $j + 1)
 		echo "Benchmark diff $DIFF_NAME file-diff ($NOW_COUNT/$RUN_NUM)"
-		./diff $LOWER $UPPER diff_file_$DIFF_NAME diff_file_$DIFF_NAME.json file-diff benchmark
+		$BIN_DIFF $LOWER $UPPER diff_file_$DIFF_NAME diff_file_$DIFF_NAME.json file-diff benchmark
 	done
 
 	# packing diff data and test it
 	echo "Testing packed $DIFF_NAME file-diff"
-	./pack diff_file_$DIFF_NAME diff_file_$DIFF_NAME.json $LOWER-base.dimg diff_file_$DIFF_NAME.dimg
+	$BIN_PACK diff_file_$DIFF_NAME diff_file_$DIFF_NAME.json $LOWER-base.dimg diff_file_$DIFF_NAME.dimg
 	ls -l diff_file_$DIFF_NAME.dimg
-	./patch dimg $LOWER $UPPER-patched diff_file_$DIFF_NAME.dimg
+	$BIN_PATCH dimg $LOWER $UPPER-patched diff_file_$DIFF_NAME.dimg
 	diff -r $UPPER $UPPER-patched --no-dereference
 done
 
@@ -90,32 +103,32 @@ MERGED=$IMAGE_LOWER-$IMAGE_UPPER
 for ((j=0; j < $RUN_NUM; j++));do
 	NOW_COUNT=$(expr $j + 1)
 	echo "Benchmark merge $MERGE_LOWER and $MERGE_UPPER to $MERGED ($NOW_COUNT/$RUN_NUM)"
-	./merge dimg diff_$MERGE_LOWER.dimg diff_$MERGE_UPPER.dimg diff_merged_$MERGED.dimg benchmark
+	$BIN_MERGE dimg diff_$MERGE_LOWER.dimg diff_$MERGE_UPPER.dimg diff_merged_$MERGED.dimg benchmark
 done
 
 echo "Testing merged $MERGED"
-./patch dimg $IMAGE_LOWER $IMAGE_UPPER-merged diff_merged_$MERGED.dimg
+$BIN_PATCH dimg $IMAGE_LOWER $IMAGE_UPPER-merged diff_merged_$MERGED.dimg
 diff -r $IMAGE_UPPER $IMAGE_UPPER-merged --no-dereference
 ls -l diff_merged_$MERGED.dimg
-./fuse-diff --basedir=./$IMAGE_LOWER-base.dimg --patchdir=./diff_merged_$MERGED.dimg --mode=dimg /tmp/fuse >/dev/null 2>&1 &
+$BIN_FUSE --basedir=./$IMAGE_LOWER-base.dimg --patchdir=./diff_merged_$MERGED.dimg --mode=dimg /tmp/fuse >/dev/null 2>&1 &
 sleep 1
 diff -r $IMAGE_UPPER /tmp/fuse --no-dereference
 fusermount3 -u /tmp/fuse
 
 for ((j=0; j < $RUN_NUM; j++));do
 	NOW_COUNT=$(expr $j + 1)
 	echo "Benchmark regen-diff $MERGED binary-diff ($NOW_COUNT/$RUN_NUM)"
-	./diff $IMAGE_LOWER $IMAGE_UPPER diff_$MERGED diff_$MERGED.json binary-diff benchmark
+	$BIN_DIFF $IMAGE_LOWER $IMAGE_UPPER diff_$MERGED diff_$MERGED.json binary-diff benchmark
 done
-./pack diff_$MERGED diff_$MERGED.json $IMAGE_LOWER-base.dimg diff_$MERGED.dimg
+$BIN_PACK diff_$MERGED diff_$MERGED.json $IMAGE_LOWER-base.dimg diff_$MERGED.dimg
 ls -l diff_$MERGED.dimg
 
 for ((j=0; j < $RUN_NUM; j++));do
 	NOW_COUNT=$(expr $j + 1)
 	echo "Benchmark regen-diff $MERGED file-diff ($NOW_COUNT/$RUN_NUM)"
-	./diff $IMAGE_LOWER $IMAGE_UPPER diff_file_$MERGED diff_file_$MERGED.json file-diff benchmark
+	$BIN_DIFF $IMAGE_LOWER $IMAGE_UPPER diff_file_$MERGED diff_file_$MERGED.json file-diff benchmark
 done
-./pack diff_file_$MERGED diff_file_$MERGED.json $IMAGE_LOWER-base.dimg diff_file_$MERGED.dimg
+$BIN_PACK diff_file_$MERGED diff_file_$MERGED.json $IMAGE_LOWER-base.dimg diff_file_$MERGED.dimg
 ls -l diff_file_$MERGED.dimg
 
 echo "Benchmark done"

tests/bench_lxc.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eu
+if [ $EUID -ne 0 ]; then
+	echo "root previlige required"
+	exit 1
+fi
+
+RUN_NUM=1
+TEST=$1
+
+PATH=$PATH:/usr/local/go/bin
+
+cd $(cd $(dirname $0); pwd)
+pushd ../
+make all
+popd
+
+RESULT_DIR=benchmark
+IMAGE_DIR=$RESULT_DIR/images
+mkdir -p $IMAGE_DIR
+
+echo "Benchmarking $TEST"
+./bench_impl.sh test_$TEST.sh $IMAGE_DIR $RUN_NUM
+cp $IMAGE_DIR/$TEST/benchmark.log ./$RESULT_DIR/$TEST-benchmark.log

tests/export_lxc_image.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+IMAGE_NAME=$1
+
+sudo lxc snapshot $IMAGE_NAME snp0
+sudo lxc publish $IMAGE_NAME/snp0 --alias $IMAGE_NAME-export --compression zstd
+sudo lxc image export $IMAGE_NAME-export /tmp/$IMAGE_NAME-image