diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..78a21f1 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,216 @@ +--- +name: Test +on: + push: + branches: + - master + - main + - release/** + - gha + pull_request: null + workflow_dispatch: + +jobs: + golangci-lint: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 1 + - uses: actions/setup-go@v3 + with: + go-version: 1.21.x + - run: sudo apt-get update + - name: golangci-lint + uses: golangci/golangci-lint-action@v3.7.0 + with: + version: v1.55.2 + args: --verbose + + create-lxc-image: + name: create-lxc-image + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4.1.1 + + - uses: actions/cache/restore@v3 + id: cache-restore + with: + key: lxc-image-base-${{ hashFiles('go.sum', 'tests/setup_test.sh') }} + path: /tmp/test-image.tar.zst + lookup-only: true + + - name: setup lxd + id: s1 + if: steps.cache-restore.outputs.cache-hit != 'true' + run: ./tests/setup_lxd.sh + + - name: launch lxc container + id: s2 + if: steps.s1.conclusion == 'success' + run: ./tests/launch_test_lxc.sh + + - name: install dependencies and build + id: s3 + if: steps.s2.conclusion == 'success' + run: sudo lxc exec test -- sudo --login --user ubuntu /host/tests/setup_test.sh + + - name: export image + id: s4 + if: steps.s3.conclusion == 'success' + run: ./tests/export_lxc_image.sh test + + - uses: actions/cache/save@v3 + id: s5 + if: steps.s4.conclusion == 'success' + with: + key: lxc-image-base-${{ hashFiles('go.sum', 'tests/setup_test.sh') }} + path: /tmp/test-image.tar.zst + +# test: +# runs-on: ubuntu-22.04 +# needs: create-lxc-image +# timeout-minutes: 20 +# steps: +# - uses: actions/checkout@v4.1.1 +# - name: setup lxd +# run: ./test/setup_lxd.sh +# - uses: actions/cache/restore@v3 +# id: cache-restore +# with: +# key: lxc-image-base-${{ hashFiles('go.sum', 'test/init_test.sh') }} +# path: /tmp/test-image.tar.zst +# fail-on-cache-miss: true +# - name: load lxc image +# run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export +# - name: launch lxc container +# run: ./test/launch_test_lxc.sh test-export +# - name: run test +# run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/bypass4netns/test/run_test.sh SYNC" +# # some source codes may be updated. re-export new image. +# - name: export image +# run: sudo lxc image alias delete test-export && rm -f /tmp/test-image.tar.zst && ./test/export_lxc_image.sh test +# - uses: actions/cache/save@v3 +# with: +# key: lxc-image-${{ github.sha }} +# path: /tmp/test-image.tar.zst +# #- name: debug +# # run: ./debug.sh +# +# benchmark: +# runs-on: ubuntu-22.04 +# needs: test +# timeout-minutes: 20 +# strategy: +# matrix: +# script: ["iperf3/iperf3_host", "iperf3/iperf3", "postgres/postgres", "redis/redis", "block/block", "memcached/memcached", "rabbitmq/rabbitmq", "etcd/etcd", "mysql/mysql"] +# steps: +# - uses: actions/checkout@v4.1.1 +# - name: setup lxd +# run: ./test/setup_lxd.sh +# - uses: actions/cache/restore@v3 +# id: cache-restore +# with: +# key: lxc-image-${{ github.sha }} +# path: /tmp/test-image.tar.zst +# fail-on-cache-miss: true +# - name: load lxc image +# run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export +# - name: launch lxc container +# run: ./test/launch_test_lxc.sh test-export +# - name: run benchmark (${{ matrix.script }}) +# run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}.sh" +# - name: upload plot +# id: get_plot +# if: matrix.script != 'iperf3/iperf3_host' +# run: | +# mkdir /tmp/benchmark-results +# sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-rootful-direct.log /tmp/benchmark-results/. +# sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-rootful-host.log /tmp/benchmark-results/. +# sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-wo-b4ns-direct.log /tmp/benchmark-results/. +# sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-wo-b4ns-host.log /tmp/benchmark-results/. +# sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-w-b4ns.log /tmp/benchmark-results/. +# - uses: actions/upload-artifact@v3 +# if: steps.get_plot.conclusion == 'success' +# with: +# name: benchmark-results +# path: /tmp/benchmark-results +# +# +# benchmark-multinode: +# runs-on: ubuntu-22.04 +# needs: test +# timeout-minutes: 20 +# strategy: +# matrix: +# script: ["iperf3/iperf3", "postgres/postgres", "redis/redis", "block/block", "memcached/memcached", "rabbitmq/rabbitmq", "etcd/etcd", "mysql/mysql"] +# steps: +# - uses: actions/checkout@v4.1.1 +# - name: setup lxd +# run: ./test/setup_lxd.sh +# - uses: actions/cache/restore@v3 +# id: cache-restore +# with: +# key: lxc-image-${{ github.sha }} +# path: /tmp/test-image.tar.zst +# fail-on-cache-miss: true +# - name: load lxc image +# run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export +# - name: launch lxc container +# run: ./test/launch_test_lxc.sh test-export +# - name: run benchmark (${{ matrix.script }}) +# run: ./benchmark/${{ matrix.script }}_multinode.sh +# - name: upload plot +# run: | +# mkdir /tmp/benchmark-results +# cp benchmark/${{ matrix.script }}-multinode-rootful.log /tmp/benchmark-results/. +# cp benchmark/${{ matrix.script }}-multinode-wo-b4ns.log /tmp/benchmark-results/. +# cp benchmark/${{ matrix.script }}-multinode-w-b4ns.log /tmp/benchmark-results/. +# - uses: actions/upload-artifact@v3 +# with: +# name: benchmark-results +# path: /tmp/benchmark-results +# +# plot: +# runs-on: ubuntu-22.04 +# needs: [benchmark, benchmark-multinode] +# steps: +# - uses: actions/checkout@v4.1.1 +# - run: sudo apt update && sudo apt install python3 python3-pip +# - run: pip3 install matplotlib numpy +# - uses: actions/download-artifact@v3 +# with: +# name: benchmark-results +# path: ./ +# - run: mkdir /tmp/benchmark-plots +# - run: python3 benchmark/redis/redis_plot.py redis-rootful-direct.log redis-rootful-host.log redis-wo-b4ns-direct.log redis-wo-b4ns-host.log redis-w-b4ns.log /tmp/benchmark-plots/redis.png +# - run: python3 benchmark/redis/redis_plot.py redis-multinode-rootful.log redis-multinode-wo-b4ns.log redis-multinode-w-b4ns.log /tmp/benchmark-plots/redis-multinode.png +# - run: python3 benchmark/iperf3/iperf3_plot.py iperf3-rootful-direct.log iperf3-rootful-host.log iperf3-wo-b4ns-direct.log iperf3-wo-b4ns-host.log iperf3-w-b4ns.log /tmp/benchmark-plots/iperf3.png +# - run: python3 benchmark/iperf3/iperf3_plot.py iperf3-multinode-rootful.log iperf3-multinode-wo-b4ns.log iperf3-multinode-w-b4ns.log /tmp/benchmark-plots/iperf3-multinode.png +# - run: python3 benchmark/postgres/postgres_plot.py postgres-rootful-direct.log postgres-rootful-host.log postgres-wo-b4ns-direct.log postgres-wo-b4ns-host.log postgres-w-b4ns.log /tmp/benchmark-plots/postgres.png +# - run: python3 benchmark/postgres/postgres_plot.py postgres-multinode-rootful.log postgres-multinode-wo-b4ns.log postgres-multinode-w-b4ns.log /tmp/benchmark-plots/postgres-multinode.png +# - run: python3 benchmark/block/block_plot.py block-rootful-direct.log block-rootful-host.log block-wo-b4ns-direct.log block-wo-b4ns-host.log block-w-b4ns.log /tmp/benchmark-plots/block.png +# - run: python3 benchmark/block/block_plot.py block-multinode-rootful.log block-multinode-wo-b4ns.log block-multinode-w-b4ns.log /tmp/benchmark-plots/block-multinode.png +# - run: python3 benchmark/memcached/memcached_plot.py memcached-rootful-direct.log memcached-rootful-host.log memcached-wo-b4ns-direct.log memcached-wo-b4ns-host.log memcached-w-b4ns.log /tmp/benchmark-plots/memcached.png +# - run: python3 benchmark/memcached/memcached_plot.py memcached-multinode-rootful.log memcached-multinode-wo-b4ns.log memcached-multinode-w-b4ns.log /tmp/benchmark-plots/memcached-multinode.png +# - run: python3 benchmark/rabbitmq/rabbitmq_plot.py rabbitmq-rootful-direct.log rabbitmq-rootful-host.log rabbitmq-wo-b4ns-direct.log rabbitmq-wo-b4ns-host.log rabbitmq-w-b4ns.log /tmp/benchmark-plots/rabbitmq.png +# - run: python3 benchmark/rabbitmq/rabbitmq_plot.py rabbitmq-multinode-rootful.log rabbitmq-multinode-wo-b4ns.log rabbitmq-multinode-w-b4ns.log /tmp/benchmark-plots/rabbitmq-multinode.png +# - run: python3 benchmark/etcd/etcd_plot.py etcd-rootful-direct.log etcd-rootful-host.log etcd-wo-b4ns-direct.log etcd-wo-b4ns-host.log etcd-w-b4ns.log /tmp/benchmark-plots/etcd.png +# - run: python3 benchmark/etcd/etcd_plot.py etcd-multinode-rootful.log etcd-multinode-wo-b4ns.log etcd-multinode-w-b4ns.log /tmp/benchmark-plots/etcd-multinode.png +# - run: python3 benchmark/mysql/mysql_plot.py mysql-rootful-direct.log mysql-rootful-host.log mysql-wo-b4ns-direct.log mysql-wo-b4ns-host.log mysql-w-b4ns.log /tmp/benchmark-plots/mysql.png +# - run: python3 benchmark/mysql/mysql_plot.py mysql-multinode-rootful.log mysql-multinode-wo-b4ns.log mysql-multinode-w-b4ns.log /tmp/benchmark-plots/mysql-multinode.png +# - uses: actions/upload-artifact@v3 +# with: +# name: benchmark-plots +# path: /tmp/benchmark-plots +# +# bench-script: +# runs-on: ubuntu-22.04 +# steps: +# - uses: actions/checkout@v4.1.1 +# - run: sudo ip a add 192.168.6.2/32 dev eth0 +# - run: hostname -I +# - run: ./test/init_test.sh +# - run: ~/bypass4netns/benchmark/run_bench.sh diff --git a/tests/.gitignore b/tests/.gitignore index 08df9cf..234b1e4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -2,10 +2,7 @@ !/.gitignore !/postgres !/simple-dir -!/run-tests.sh -!/bench*.sh !/bench-*.py -!/push-*.sh -!/scp*.sh -!version.sh +!/*.sh +!/lxd.yaml /server diff --git a/tests/export_lxc_image.sh b/tests/export_lxc_image.sh new file mode 100755 index 0000000..0467b83 --- /dev/null +++ b/tests/export_lxc_image.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eux -o pipefail + +IMAGE_NAME=$1 + +sudo lxc snapshot $IMAGE_NAME snp0 +sudo lxc publish $IMAGE_NAME/snp0 --alias $IMAGE_NAME-export --compression zstd +sudo lxc image export $IMAGE_NAME-export /tmp/$IMAGE_NAME-image diff --git a/tests/launch_test_lxc.sh b/tests/launch_test_lxc.sh new file mode 100755 index 0000000..10bac00 --- /dev/null +++ b/tests/launch_test_lxc.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ux -o pipefail +IMAGE=${1:-"images:ubuntu/22.04"} + +cd $(cd $(dirname $0); pwd) + +sudo lxc launch -c security.privileged=true -c security.nesting=true $IMAGE test +sudo lxc config device add test share disk source=$(cd ../; pwd) path=/host +sudo lxc exec test -- /bin/bash -c "echo 'ubuntu ALL=NOPASSWD: ALL' | EDITOR='tee -a' visudo" + +# let user services running +# this sometimes fails, retry until success +RES=1 +while [ $RES -ne 0 ] +do + sleep 1 + sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sudo loginctl enable-linger" + RES=$? +done diff --git a/tests/lxd.yaml b/tests/lxd.yaml new file mode 100644 index 0000000..26fc8b0 --- /dev/null +++ b/tests/lxd.yaml @@ -0,0 +1,38 @@ +config: {} +networks: +- config: + ipv4.address: 192.168.6.1/24 + ipv4.nat: "true" + ipv6.address: none + description: "" + name: lxdbr0 + type: bridge + project: default +storage_pools: +- config: + size: 30GiB + source: /var/snap/lxd/common/lxd/disks/default.img + description: "" + name: default + driver: btrfs +profiles: +- config: {} + description: Default LXD profile + devices: + eth0: + name: eth0 + network: lxdbr0 + type: nic + root: + path: / + pool: default + type: disk + name: default +projects: +- config: + features.images: "true" + features.networks: "true" + features.profiles: "true" + features.storage.volumes: "true" + description: Default LXD project + name: default \ No newline at end of file diff --git a/tests/setup_lxd.sh b/tests/setup_lxd.sh new file mode 100755 index 0000000..3c562d9 --- /dev/null +++ b/tests/setup_lxd.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eux -o pipefail + +cd $(cd $(dirname $0); pwd) + +sudo modprobe vxlan +cat lxd.yaml | sudo lxd init --preseed +sudo sysctl -w net.ipv4.ip_forward=1 + +#https://andreas.scherbaum.la/post/2023-01-18_fix-lxc-network-issues-in-ubuntu-22.04/ +sudo iptables -I DOCKER-USER -i lxdbr0 -o eth0 -j ACCEPT +sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +sudo iptables -F FORWARD +sudo iptables -P FORWARD ACCEPT \ No newline at end of file diff --git a/tests/setup_test.sh b/tests/setup_test.sh new file mode 100755 index 0000000..95d267f --- /dev/null +++ b/tests/setup_test.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +TEST_USER=ubuntu +set -eu -o pipefail + +if [ "$(whoami)" != "$TEST_USER" ]; then + su $TEST_USER -c $0 + exit 0 +fi + +GO_VERSION="1.21.8" + +echo "===== Prepare =====" +( + set -x + + # for lxc + if [ -d /host ]; then + sudo cp -r /host ~/d4c + fi + + sudo chown -R $TEST_USER:$TEST_USER ~/d4c + + sudo apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + sudo apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y dbus-user-session docker-ce containerd.io golang-go fuse3 python3 python3-pip + pip3 install docker-squash + #pip3 install matplotlib numpy + + systemctl --user start dbus + + curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz | sudo tar Cxz /usr/local + echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.profile + source ~/.profile + + ## build nerdctl with bypass4netns + #curl -fsSL https://github.com/containerd/nerdctl/archive/refs/tags/v${NERDCTL_VERSION}.tar.gz | tar Cxz ~/ + #cd ~/nerdctl-${NERDCTL_VERSION} + #echo "replace github.com/rootless-containers/bypass4netns => /home/$TEST_USER/bypass4netns" >> go.mod + #make + #sudo rm -f /usr/local/bin/nerdctl + #sudo cp _output/nerdctl /usr/local/bin/nerdctl + #nerdctl info + + #cd ~/bypass4netns + #make + #sudo rm -f /usr/local/bin/bypass4netns* + #sudo make install + + ## also enable rootful containerd for rootful container testing + sudo systemctl enable --now containerd +)