diff --git a/.github/workflows/containerfiles/Dockerfile.percona-debian-build b/.github/workflows/containerfiles/Dockerfile.percona-debian-build new file mode 100644 index 00000000..af128ca0 --- /dev/null +++ b/.github/workflows/containerfiles/Dockerfile.percona-debian-build @@ -0,0 +1,111 @@ +FROM docker.io/ubuntu:20.04 + +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + build-essential \ + git \ + pkg-config \ + cmake \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + zlib1g-dev \ + valgrind \ + libssl-dev \ + libncurses-dev \ + libldap2-dev \ + libreadline-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + bison \ + wget \ + zlib1g-dev \ + valgrind \ + libssl1.1 \ + libncurses6 \ + libncursesw6 \ + libldap-2.4-2 \ + libreadline8 \ + libcurl4 \ + libgflags2.2 \ + perl \ + curl \ + lsb-release \ + debsums \ + dialog \ + apt-utils \ + libaio1 \ + libmecab2 \ + libnuma1 \ + psmisc \ + libjemalloc2 \ + libjemalloc-dev \ + python2 \ + build-essential \ + python-dev \ + sudo \ + automake \ + libtool \ + pkg-config \ + libaio-dev \ + && apt-get clean + +ENV WORK_PATH=/percona/work +ENV INSTALL_PATH=/percona/dest +ENV BUILD_PATH=/tmp/BUILD_PS +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH +RUN mkdir -p $BUILD_PATH + +WORKDIR $WORK_PATH + +ARG PERCONA_SERVER_BASE +ARG ZENFS_REPO +ARG ZENFS_BRANCH + +# Following the percona blog post to build the debian packages: https://www.percona.com/blog/2021/03/10/how-to-build-percona-server-for-mysql-from-sources/ + +RUN wget https://raw.githubusercontent.com/percona/percona-server/8.0/build-ps/percona-server-8.0_builder.sh +RUN sed -i 's=#!/bin/sh=#!/bin/bash=' percona-server-8.0_builder.sh +RUN chmod +x percona-server-8.0_builder.sh +ENV DEB_RELEASE_EXTRA=experimentalZenFS +RUN ./percona-server-8.0_builder.sh --builddir=${BUILD_PATH} --install_deps=1 --with_zenfs=1 --deb_release=${DEB_RELEASE_EXTRA} +# PERCONA_SERVER_BASE must have an origin in a Percona-Server-* tag. +RUN ./percona-server-8.0_builder.sh --builddir=${BUILD_PATH} --get_sources=1 --branch=${PERCONA_SERVER_BASE} --with_zenfs=1 --zenfs_branch=${ZENFS_BRANCH} --zenfs_repo=${ZENFS_REPO} --deb_release=${DEB_RELEASE_EXTRA} +RUN ./percona-server-8.0_builder.sh --builddir=${BUILD_PATH} --build_source_deb=1 --with_zenfs=1 --deb_release=${DEB_RELEASE_EXTRA} +RUN ./percona-server-8.0_builder.sh --builddir=${BUILD_PATH} --build_deb=1 --with_zenfs=1 --deb_release=${DEB_RELEASE_EXTRA} + +ENV PATH="$INSTALL_PATH/bin:$PATH" +ENV LD_LIBRARY_PATH="$INSTALL_PATH/lib" + +RUN wget https://repo.percona.com/apt/pool/main/j/jemalloc/libjemalloc1_3.6.0-2.focal_amd64.deb +RUN dpkg -i libjemalloc1_3.6.0-2.focal_amd64.deb + +WORKDIR $BUILD_PATH +RUN dpkg -i *.deb + +RUN find ${BUILD_PATH} -type f -name zenfs | xargs -I {} cp {} ${INSTALL_PATH} +RUN find ${BUILD_PATH} -type f -name mysql-test-run.pl | grep builddir | xargs -I {} ln -s {} ${INSTALL_PATH}/mtr + +RUN apt-get update \ + && \ + apt-get install --yes --quiet=2 \ + libmysqlclient-dev \ + libssl-dev \ + python3-mysqldb \ + && apt-get clean + +WORKDIR $WORK_PATH +RUN git clone https://github.com/akopytov/sysbench.git \ + && cd sysbench \ + && ./autogen.sh \ + && ./configure \ + && make -j \ + && make install \ + && cd - + +WORKDIR $BUILD_PATH diff --git a/.github/workflows/containerfiles/Dockerfile.percona-server b/.github/workflows/containerfiles/Dockerfile.percona-server new file mode 100644 index 00000000..e73ebd60 --- /dev/null +++ b/.github/workflows/containerfiles/Dockerfile.percona-server @@ -0,0 +1,99 @@ +FROM docker.io/ubuntu:22.04 AS builder + +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + build-essential \ + git \ + pkg-config \ + cmake \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + zlib1g-dev \ + valgrind \ + libssl-dev \ + libncurses-dev \ + libldap2-dev \ + libreadline-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + bison \ + && apt-get clean + +ENV WORK_PATH=/percona/work +ENV INSTALL_PATH=/percona/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +ARG LIBZBD_GIT_CHECKOUT +RUN git clone --quiet --single-branch --branch master -n --quiet http://github.com/westerndigitalcorporation/libzbd.git \ + && cd libzbd \ + && git checkout $LIBZBD_GIT_CHECKOUT +RUN cd libzbd && sh autogen.sh && ./configure --prefix=${INSTALL_PATH} && make install + +COPY percona-server percona-server +RUN PKG_CONFIG_PATH=$INSTALL_PATH/lib/pkgconfig \ + cmake ./percona-server \ + -DUSE_VALGRIND=ON \ + -DCMAKE_C_FLAGS="-mno-avx512f" \ + -DCMAKE_CXX_FLAGS="-mno-avx512f" \ + -DWITH_AUTHENTICATION_KERBEROS=OFF \ + -DWITH_AUTHENTICATION_LDAP=OFF \ + -DWITH_KERBEROS=none \ + -DWITH_ROCKSDB=ON \ + -DROCKSDB_PLUGINS=zenfs \ + -DWITH_ZENFS_UTILITY=ON \ + -DDOWNLOAD_BOOST=1\ + -DWITH_BOOST=$WORK_PATH/boost \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} \ + && make -j$(nproc) \ + && make install + +FROM docker.io/ubuntu:22.04 +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + build-essential \ + zlib1g-dev \ + valgrind \ + libssl3 \ + libssl-dev \ + libncurses6 \ + libncursesw6 \ + libldap2-dev \ + libreadline8 \ + libcurl4 \ + libgflags2.2 \ + libmysqlclient-dev \ + perl \ + python3-dev \ + python3-pip \ + python-is-python3 \ + pkg-config \ + && apt-get clean + +ENV WORK_PATH=/percona/work +ENV INSTALL_PATH=/percona/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +COPY --from=builder $INSTALL_PATH $INSTALL_PATH +# Changeing access permissions of directories in $INSTALL_PATH is needed for the mysql-test-run.pl (see line 2415 and 3053) +RUN find $INSTALL_PATH -type d -exec chmod 777 {} + +RUN adduser --quiet --disabled-password mysql +RUN apt-get update && apt-get install -y --quiet sudo + +ENV PATH="$INSTALL_PATH/bin:$PATH" +ENV LD_LIBRARY_PATH="$INSTALL_PATH/lib" + +RUN export MYSQLCLIENT_CFLAGS=$(pkg-config mysqlclient --cflags); export MYSQLCLIENT_LDFLAGS=$(pkg-config mysqlclient --libs); pip install mysqlclient; diff --git a/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs b/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs new file mode 100644 index 00000000..63348ca8 --- /dev/null +++ b/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs @@ -0,0 +1,90 @@ +FROM docker.io/ubuntu:20.04 AS builder + +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + build-essential \ + git \ + pkg-config \ + cmake \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + libgflags-dev \ + libsnappy-dev \ + uuid-dev \ + bison \ + autopoint \ + && apt-get clean + +ENV WORK_PATH=/rocks/work +ENV INSTALL_PATH=/rocks/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +ARG LIBZBD_GIT_CHECKOUT +RUN git clone --quiet --single-branch --branch master -n --quiet http://github.com/westerndigitalcorporation/libzbd.git \ + && cd libzbd \ + && git checkout $LIBZBD_GIT_CHECKOUT +RUN cd libzbd && sh autogen.sh && ./configure --prefix=${INSTALL_PATH} && make install + +RUN git clone --depth 1 --single-branch --branch v2.37.2 https://github.com/util-linux/util-linux \ + && cd util-linux \ + && ./autogen.sh \ + && ./configure --prefix=$INSTALL_PATH \ + && make blkzone \ + && install blkzone $INSTALL_PATH/bin + +COPY rocksdb rocksdb +ARG ROCKSDB_DEBUG_LEVEL +RUN cd rocksdb \ + && DEBUG_LEVEL=$ROCKSDB_DEBUG_LEVEL \ + ROCKSDB_PLUGINS=zenfs \ + PREFIX=$INSTALL_PATH \ + PKG_CONFIG_PATH=$INSTALL_PATH/lib/pkgconfig \ + OBJ_DIR=/tmp/zenfs-build \ + make -j$(nproc) db_bench db_stress ldb \ + && DEBUG_LEVEL=$ROCKSDB_DEBUG_LEVEL \ + ROCKSDB_PLUGINS=zenfs \ + PREFIX=$INSTALL_PATH \ + PKG_CONFIG_PATH=$INSTALL_PATH/lib/pkgconfig \ + OBJ_DIR=/tmp/zenfs-build \ + make -j$(nproc) install \ + && install db_bench $INSTALL_PATH/bin \ + && install db_stress $INSTALL_PATH/bin \ + && install ldb $INSTALL_PATH/bin \ + && cd plugin/zenfs/util \ + && PKG_CONFIG_PATH=$INSTALL_PATH/lib/pkgconfig make \ + && install zenfs $INSTALL_PATH/bin + +FROM docker.io/ubuntu:20.04 +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + libgflags2.2 \ + libsnappy1v5 \ + python3 \ + python3-distutils \ + python3-matplotlib \ + nvme-cli \ + && apt-get clean + +ENV WORK_PATH=/rocks/work +ENV INSTALL_PATH=/rocks/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +COPY --from=builder $INSTALL_PATH $INSTALL_PATH +COPY --from=builder $WORK_PATH/rocksdb/plugin/zenfs/tests $INSTALL_PATH/zenfs/test + +ENV PATH="$INSTALL_PATH/bin:$PATH" +ENV LD_LIBRARY_PATH="$INSTALL_PATH/lib" diff --git a/.github/workflows/containerfiles/Dockerfile.terarkdb-zenfs b/.github/workflows/containerfiles/Dockerfile.terarkdb-zenfs new file mode 100644 index 00000000..0ad12114 --- /dev/null +++ b/.github/workflows/containerfiles/Dockerfile.terarkdb-zenfs @@ -0,0 +1,97 @@ +FROM docker.io/ubuntu:20.04 AS builder + +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + build-essential \ + git \ + pkg-config \ + cmake \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + libgflags-dev \ + libsnappy-dev \ + uuid-dev \ + bison \ + autopoint \ + libaio-dev \ + && apt-get clean + +ENV WORK_PATH=/rocks/work +ENV INSTALL_PATH=/rocks/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +ARG LIBZBD_GIT_CHECKOUT +RUN git clone --quiet --single-branch --branch master -n --quiet http://github.com/westerndigitalcorporation/libzbd.git \ + && cd libzbd \ + && git checkout $LIBZBD_GIT_CHECKOUT +RUN cd libzbd && sh autogen.sh && ./configure --prefix=${INSTALL_PATH} && make install + +RUN git clone --depth 1 --single-branch --branch v2.37.2 https://github.com/util-linux/util-linux \ + && cd util-linux \ + && ./autogen.sh \ + && ./configure --prefix=$INSTALL_PATH \ + && make blkzone \ + && install blkzone $INSTALL_PATH/bin + +COPY terarkdb terarkdb + +WORKDIR $WORK_PATH/terark-build + +RUN \ + PKG_CONFIG_PATH=$INSTALL_PATH/lib/pkgconfig \ + cmake $WORK_PATH/terarkdb \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PATH \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWITH_TESTS=OFF \ + -DWITH_ZENFS=ON \ + -DBYTEDANCE_METRICS_PATH=OFF \ + -DWITH_BYTEDANCE_METRICS=OFF \ + -DWITH_TOOLS=ON \ + -DWITH_TERARK_ZIP=OFF \ + -DCMAKE_C_FLAGS="-I$INSTALL_PATH/include" \ + -DCMAKE_CXX_FLAGS="-I$INSTALL_PATH/include" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L$INSTALL_PATH/lib" \ + -DCMAKE_EXE_LINKER_FLAGS="-L$INSTALL_PATH/lib" + +RUN make -j $(nproc) VERBOSE=1 +RUN make install + +RUN \ + install db_bench $INSTALL_PATH/bin \ + && install zenfs $INSTALL_PATH/bin + +FROM docker.io/ubuntu:20.04 +RUN apt-get update \ + && \ + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + apt-get install --yes --quiet=2 \ + libgflags2.2 \ + libsnappy1v5 \ + python3 \ + python3-distutils \ + python3-matplotlib \ + nvme-cli \ + libaio1 \ + && apt-get clean + +ENV WORK_PATH=/rocks/work +ENV INSTALL_PATH=/rocks/dest +RUN mkdir -p $WORK_PATH +RUN mkdir -p $INSTALL_PATH + +WORKDIR $WORK_PATH + +COPY --from=builder $INSTALL_PATH $INSTALL_PATH +COPY --from=builder $WORK_PATH/terarkdb/third-party/zenfs/tests $INSTALL_PATH/zenfs/test + +ENV PATH="$INSTALL_PATH/bin:$PATH" +ENV LD_LIBRARY_PATH="$INSTALL_PATH/lib" diff --git a/.github/workflows/cpp-lint.yaml b/.github/workflows/cpp-lint.yaml index 3bb1a9c6..e2c145fc 100644 --- a/.github/workflows/cpp-lint.yaml +++ b/.github/workflows/cpp-lint.yaml @@ -11,7 +11,7 @@ jobs: name: Lint with clang-format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run clang-format style check for C/C++ programs. uses: jidicula/clang-format-action@v4.10.1 with: diff --git a/.github/workflows/long-performance-xfs.yaml b/.github/workflows/long-performance-xfs.yaml index 10f77f3e..4d3d65df 100644 --- a/.github/workflows/long-performance-xfs.yaml +++ b/.github/workflows/long-performance-xfs.yaml @@ -10,31 +10,81 @@ jobs: steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs - - name: Run long performance test 1TB - run: cd harness && st --silent -o logs/make -- sudo with-xfs --disk 1TB -- make NO_VAGRANT=1 results/xfs-long.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload + path: rocksdb/plugin/zenfs + + - name: Create results directory + run: | + mkdir -p results + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Format conventional block device + run: | + sudo nvme format /dev/${BDEV0} -f --ses=1 + + - name: Create xfs on BDEV + run: | + sudo mkfs.xfs -f /dev/${BDEV0} + + - name: Create xfs mountpoint + run: | + mkdir -p xfs-long-performance + + - name: Mount BDEV to xfs mountpoint + run: | + sudo mount /dev/${BDEV0} xfs-long-performance + + - name: Change back ownership of mountpoint + run: | + sudo chown $(id -u):$(id -g) xfs-long-performance + + - name: Run long performance test + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ./results:/rocks/work/results:Z \ + --volume ./xfs-long-performance:/mnt/xfs:Z \ + --security-opt unmask=/sys/dev/block \ + rocksdb-zenfs \ + bash -c "FS_PARAMS='--db=/mnt/xfs --target_file_size_base=4080218931 --use_direct_io_for_flush_and_compaction --max_bytes_for_level_multiplier=4 --max_background_jobs=8 --use_direct_reads --write_buffer_size=2147483648' PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-xfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) long_performance" + + - name: Archive long performance test results + uses: actions/upload-artifact@v3 + with: + name: long_performance + path: ./results + if: always() + + - name: Remove result dir + run: rm -rf results + if: always() + + - name: Umount xfs mountpoint + run: sudo umount -f xfs-long-performance + if: always() + + - name: Remove xfs mountpoint + run: rm -rf xfs-long-performance if: always() + - name: Remove images run: podman rmi --force --all if: always() - diff --git a/.github/workflows/long-performance.yaml b/.github/workflows/long-performance.yaml index fdec304f..7c23175b 100644 --- a/.github/workflows/long-performance.yaml +++ b/.github/workflows/long-performance.yaml @@ -7,38 +7,70 @@ name: Long Performance Test jobs: long-performance: name: Long Performance - runs-on: self-hosted + runs-on: nvme-zns timeout-minutes: 4320 strategy: matrix: +#TODO: Figure out a way to assign drive sizes -> maybe with env-vars? drive-size: [1TB] steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs - - name: Run long performance test ${{ matrix.drive-size }} - run: cd harness && st --silent -o logs/make -- disk-select --disk ${{ matrix.drive-size }} -- make NO_VAGRANT=1 results/zenfs-long.xml - - name: Collect Results ${{ matrix.drive-size }} - run: cd harness && make NO_VAGRANT=1 upload + path: rocksdb/plugin/zenfs + + - name: Create results directory + run: | + mkdir -p results + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + + - name: RocksDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" + + - name: Run long performance test + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ./results:/rocks/work/results:Z \ + --security-opt unmask=/sys/dev/block \ + rocksdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results DB_BENCH_EXTRA_PARAMS=$(/rocks/dest/zenfs/test/get_good_db_bench_params_for_zenfs.sh ${ZBD0}) /rocks/dest/zenfs/test/run.sh rocksdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) long_performance" + + - name: Archive long performance test results + uses: actions/upload-artifact@v3 + with: + name: long_performance + path: ./results + + - name: Remove result dir + run: rm -rf results if: always() + - name: Remove images run: podman rmi --force --all if: always() diff --git a/.github/workflows/mtr-test.yaml b/.github/workflows/mtr-test.yaml index a91a1002..3985ae1d 100644 --- a/.github/workflows/mtr-test.yaml +++ b/.github/workflows/mtr-test.yaml @@ -1,3 +1,4 @@ +#TODO: make sure to run on 1TB zns drives -> look at disk-select option on: workflow_dispatch: schedule: @@ -5,58 +6,111 @@ on: name: MTR Tests jobs: mtr-tests: - runs-on: self-hosted + runs-on: nvme-zns timeout-minutes: 1440 steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout percona-server - uses: actions/checkout@v2 - with: - repository: percona/percona-server - ref: release-8.0.30-22 - path: harness/percona-context/percona-server - submodules: true - - name: Remove default zenfs - run: rm -r harness/percona-context/percona-server/storage/rocksdb/rocksdb_plugins/zenfs - - name: Checkout recent zenfs - uses: actions/checkout@v2 + + - name: Prepare zoned block devices + run: | + for i in $(seq 0 3) + do + DEV="ZBD$i" + if [ ! -b "/dev/${!DEV}" ] + then + echo "/dev/${!DEV} is not available for zenfs-mtr!" + exit 1 + fi + echo deadline > /sys/block/${!DEV}/queue/scheduler + done + + - name: Checkout zenfs + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/percona-context/percona-server/storage/rocksdb/rocksdb_plugins/zenfs + path: zenfs + + - name: Create build directory + run: | + mkdir -p percona-build-dir + + - name: Create result directory + run: | + mkdir -p percona-results + + - name: Build docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t percona-debian \ + -f zenfs/.github/workflows/containerfiles/Dockerfile.percona-debian-build \ + --build-arg PERCONA_SERVER_BASE=Percona-Server-8.0.34-26 \ + --build-arg ZENFS_BRANCH=${GITHUB_SHA} \ + --build-arg ZENFS_REPO=https://github.com/${GITHUB_REPOSITORY}.git \ + . + - name: Run MTR RocksDB test id: mtrTest1 - run: cd harness && st --silent -o logs/makeMtrRocksdb -- disk-select --disk 1TB -- make NO_VAGRANT=1 RUN_ARGS="mtrRocksdbTest" results/zenfs-mtr.xml + run: | + podman run --rm -i --device /dev/$ZBD0 \ + --volume ${GITHUB_WORKSPACE}/zenfs/.github/workflows/mtr:/scripts \ + --workdir /scripts \ + --env DEVICE0="$ZBD0" \ + percona-debian \ + bash wrapper.sh bash mtr-rocksdb-test.sh continue-on-error: true - - name: Collect MTR RocksDB results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + - name: Run MTR RocksDB replication test id: mtrTest2 - run: cd harness && st --silent -o logs/makeMtrRocksdbReplication -- disk-select --disk 1TB --disk 1TB -- make NO_VAGRANT=1 RUN_ARGS="mtrRocksdbReplicationTest" results/zenfs-mtr.xml + run: | + podman run --rm -i --device /dev/$ZBD0 \ + --device /dev/$ZBD1 \ + --volume ${GITHUB_WORKSPACE}/zenfs/.github/workflows/mtr:/scripts \ + --workdir /scripts \ + --env DEVICE0="$ZBD0" \ + --env DEVICE1="$ZBD1" \ + percona-debian \ + bash wrapper.sh bash mtr-rocksdb-replication-test.sh continue-on-error: true - - name: Collect MTR RocksDB replication results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + - name: Run MTR RocksDB parallel test id: mtrTest3 - run: cd harness && st --silent -o logs/makeMtrRocksdbParallel -- disk-select --disk 1TB --disk 1TB --disk 1TB --disk 1TB -- make NO_VAGRANT=1 RUN_ARGS="mtrRocksdbParallelTest" results/zenfs-mtr.xml + run: | + podman run --rm -i --device /dev/$ZBD0 \ + --device /dev/$ZBD1 \ + --device /dev/$ZBD2 \ + --device /dev/$ZBD3 \ + --volume ${GITHUB_WORKSPACE}/zenfs/.github/workflows/mtr:/scripts \ + --workdir /scripts \ + --env DEVICE0="$ZBD0" \ + --env DEVICE1="$ZBD1" \ + --env DEVICE2="$ZBD2" \ + --env DEVICE3="$ZBD3" \ + percona-debian \ + bash wrapper.sh bash mtr-rocksdb-parallel-test.sh continue-on-error: true - - name: Collect MTR RocksDB parallel results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + - name: Run MTR RocksDB parallel replication test id: mtrTest4 - run: cd harness && st --silent -o logs/makeMtrRocksdbParallelReplication -- disk-select --disk 1TB --disk 1TB --disk 1TB --disk 1TB -- make NO_VAGRANT=1 RUN_ARGS="mtrRocksdbParallelReplicationTest" results/zenfs-mtr.xml + run: | + podman run --rm -i --device /dev/$ZBD0 \ + --device /dev/$ZBD1 \ + --device /dev/$ZBD2 \ + --device /dev/$ZBD3 \ + --volume ${GITHUB_WORKSPACE}/zenfs/.github/workflows/mtr:/scripts \ + --workdir /scripts \ + --env DEVICE0="$ZBD0" \ + --env DEVICE1="$ZBD1" \ + --env DEVICE2="$ZBD2" \ + --env DEVICE3="$ZBD3" \ + percona-debian \ + bash wrapper.sh bash mtr-rocksdb-parallel-replication-test.sh continue-on-error: true - - name: Collect MTR RocksDB parallel replication results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + - name: Remove images run: podman image prune --force if: always() + - name: Failure check - if: steps.mtrTest1.outcome != 'success' || steps.mtrTest2.outcome != 'success' || steps.mtrTest3.outcome != 'success' || steps.mtrTest4.outcome != 'success' + if: steps.mtrTest1.outcome != 'success' || steps.mtrTest2.outcome != 'success' || steps.mtrTest3.outcome != 'success' || steps.mtrTest4.outcome != 'success' run: exit 1 diff --git a/.github/workflows/mtr/expected-mtr-failures b/.github/workflows/mtr/expected-mtr-failures new file mode 100644 index 00000000..500a6812 --- /dev/null +++ b/.github/workflows/mtr/expected-mtr-failures @@ -0,0 +1,17 @@ +rocksdb.check_ignore_unknown_options : WL#0000 Need adapting +rocksdb.drop_table2 : WL#0000 Need adapting +rocksdb.rocksdb_fault_injection : WL#0000 Need adapting +rocksdb.track_and_verify_wals_in_manifest : WL#0000 Need adapting +rocksdb.type_char_index_ver_bump_verify : WL#0000 Need adapting +rocksdb.checkpoint : WL#0000 Need fixing the server +rocksdb.bloomfilter3 : WL#0000 Sporadic failures +rocksdb.bloomfilter_skip : WL#0000 Sporadic failures +rocksdb.rocksdb_read_free_rpl : WL#0000 Sporadic failures +rocksdb.optimize_table : WL#0000 Need adapting +rocksdb.issue896 : WL#0000 Sporadic failures +rocksdb.non_blocking_manual_compaction : WL#0000 Sporadic failures +rocksdb.add_index_inplace: WL#0000 Sporadic failures +rocksdb.partial_index_stress: WL#0000 TODO FIND OUT WHY THIS FAILS +rocksdb.show_table_status: WL#0000 TODO FIND OUT WHY THIS FAILS +rocksdb.rocksdb_icp: WL#0000 TODO FIND OUT WHY THIS FAILS +rocksdb.rocksdb_icp_rev: WL#0000 TODO FIND OUT WHY THIS FAILS diff --git a/.github/workflows/mtr/fs-cleanup-parallel-replication.sh b/.github/workflows/mtr/fs-cleanup-parallel-replication.sh new file mode 100755 index 00000000..c4681157 --- /dev/null +++ b/.github/workflows/mtr/fs-cleanup-parallel-replication.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +WORKER=$1 +AUX1=$((WORKER*2-2)) +AUX2=$((AUX1+1)) +SOURCE_VARNAME=SOURCE_FS_URI$WORKER +REPLICA_VARNAME=REPLICA_FS_URI$WORKER +SOURCE_FS_URI=${!SOURCE_VARNAME} +REPLICA_FS_URI=${!REPLICA_VARNAME} +SOURCE_ZBD=${SOURCE_FS_URI#zenfs://dev:} +REPLICA_ZBD=${REPLICA_FS_URI#zenfs://dev:} + +echo reinitializing ${SOURCE_ZBD} \(source\) and ${REPLICA_ZBD} \(replica\) for worker $1 + +rm -fr /tmp/zenfs_aux_$AUX1 +rm -fr /tmp/zenfs_aux_$AUX2 +zenfs mkfs --zbd=$SOURCE_ZBD --aux-path=/tmp/zenfs_aux_$AUX1 --force +zenfs mkfs --zbd=$REPLICA_ZBD --aux-path=/tmp/zenfs_aux_$AUX2 --force diff --git a/.github/workflows/mtr/fs-cleanup-parallel.sh b/.github/workflows/mtr/fs-cleanup-parallel.sh new file mode 100755 index 00000000..b988ebac --- /dev/null +++ b/.github/workflows/mtr/fs-cleanup-parallel.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +WORKER=$1 +WORKER=$((WORKER-1)) +DEV="DEVICE$WORKER" +echo "Reinitializing $DEV" +if [ -z "${!DEV}" ] +then + echo "$DEV must be set for zenfs-mtr" + exit 1 +fi + +rm -fr /tmp/zenfs_aux_$WORKER +zenfs mkfs --zbd=${!DEV} --aux-path=/tmp/zenfs_aux_$WORKER --force diff --git a/.github/workflows/mtr/fs-cleanup-replication.sh b/.github/workflows/mtr/fs-cleanup-replication.sh new file mode 100755 index 00000000..15e549fd --- /dev/null +++ b/.github/workflows/mtr/fs-cleanup-replication.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +$SCRIPT_DIR/fs-multidev-cleanup.sh 1 diff --git a/.github/workflows/mtr/fs-multidev-cleanup.sh b/.github/workflows/mtr/fs-multidev-cleanup.sh new file mode 100755 index 00000000..0c08be83 --- /dev/null +++ b/.github/workflows/mtr/fs-multidev-cleanup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +if [ -z "${1}" ] +then + echo "Usage: $0 LAST_DEVICE_INDEX" + exit 1 +fi + +for i in $(seq 0 $1) +do + DEV="DEVICE$i" + echo "Reinitializing $DEV" + if [ -z "${!DEV}" ] + then + echo "$DEV must be set for zenfs-mtr" + exit 1 + fi + + rm -fr /tmp/zenfs_aux_$i + zenfs mkfs --zbd=${!DEV} --aux-path=/tmp/zenfs_aux_$i --force +done diff --git a/.github/workflows/mtr/mtr-rocksdb-parallel-replication-test.sh b/.github/workflows/mtr/mtr-rocksdb-parallel-replication-test.sh new file mode 100755 index 00000000..f4db7211 --- /dev/null +++ b/.github/workflows/mtr/mtr-rocksdb-parallel-replication-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +/scripts/fs-multidev-cleanup.sh 3 + +cd $INSTALL_PATH + +SOURCE_FS_URI1=zenfs://dev:$DEVICE0 SOURCE_FS_URI2=zenfs://dev:$DEVICE2 \ + REPLICA_FS_URI1=zenfs://dev:$DEVICE1 REPLICA_FS_URI2=zenfs://dev:$DEVICE3 \ + ./mtr --suite=rocksdb \ + --skip-test-list /scripts/expected-mtr-failures \ + --defaults-extra-file=/scripts/zenfs_rpl.cnf \ + --fs-cleanup-hook="@/scripts/fs-cleanup-parallel-replication.sh" \ + --parallel=2 diff --git a/.github/workflows/mtr/mtr-rocksdb-parallel-test.sh b/.github/workflows/mtr/mtr-rocksdb-parallel-test.sh new file mode 100755 index 00000000..36e90ee4 --- /dev/null +++ b/.github/workflows/mtr/mtr-rocksdb-parallel-test.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +/scripts/fs-multidev-cleanup.sh 3 + +cd $INSTALL_PATH + +FS_URI1=zenfs://dev:$DEVICE0 FS_URI2=zenfs://dev:$DEVICE1 \ + FS_URI3=zenfs://dev:$DEVICE2 FS_URI4=zenfs://dev:$DEVICE3 \ + ./mtr --suite=rocksdb \ + --skip-rpl \ + --skip-test-list /scripts/expected-mtr-failures \ + --defaults-extra-file=/scripts/zenfs_parallel.cnf \ + --fs-cleanup-hook="@/scripts/fs-cleanup-parallel.sh" \ + --parallel=4 diff --git a/.github/workflows/mtr/mtr-rocksdb-replication-test.sh b/.github/workflows/mtr/mtr-rocksdb-replication-test.sh new file mode 100755 index 00000000..bea4412f --- /dev/null +++ b/.github/workflows/mtr/mtr-rocksdb-replication-test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +/scripts/fs-cleanup-replication.sh + +cd $INSTALL_PATH + +SOURCE_FS_URI1=zenfs://dev:$DEVICE0 \ + REPLICA_FS_URI1=zenfs://dev:$DEVICE1 \ + ./mtr --suite=rocksdb \ + --skip-test-list /scripts/expected-mtr-failures \ + --defaults-extra-file=/scripts/zenfs_rpl.cnf \ + --fs-cleanup-hook="@/scripts/fs-cleanup-replication.sh" diff --git a/.github/workflows/mtr/mtr-rocksdb-test.sh b/.github/workflows/mtr/mtr-rocksdb-test.sh new file mode 100755 index 00000000..7ffd282c --- /dev/null +++ b/.github/workflows/mtr/mtr-rocksdb-test.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +if [ -z "${DEVICE0}" ] +then + echo "DEVICE0 not set for the test script" + exit 1 +fi + +rm -fr /tmp/zenfs_aux +zenfs mkfs --zbd=$DEVICE0 --aux-path=/tmp/zenfs_aux --force + +cd $INSTALL_PATH + +./mtr --suite=rocksdb \ + --skip-rpl \ + --skip-test-list /scripts/expected-mtr-failures \ + --mysqld=--loose-rocksdb-fs-uri=zenfs://dev:$DEVICE0 \ + --fs-cleanup-hook="rm -rf /tmp/zenfs_aux; zenfs mkfs --zbd=$DEVICE0 --aux_path=/tmp/zenfs_aux --force" diff --git a/.github/workflows/mtr/wrapper.sh b/.github/workflows/mtr/wrapper.sh new file mode 100755 index 00000000..d0ae6964 --- /dev/null +++ b/.github/workflows/mtr/wrapper.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +for i in $(seq 0 3) +do + DEV="DEVICE$i" + if [ -n "${!DEV}" ] + then + chown mysql:mysql /dev/${!DEV} + fi +done + +sudo --preserve-env=LD_LIBRARY_PATH,PATH,PERCONA_SRC,PERCONA_BUILD,PERCONA_INSTALL,DEVICE,DEVICE0,DEVICE1,DEVICE2,DEVICE3,DEVICE4,DEVICE5,INSTALL_PATH $@ diff --git a/.github/workflows/mtr/zenfs_parallel.cnf b/.github/workflows/mtr/zenfs_parallel.cnf new file mode 100644 index 00000000..32e958ce --- /dev/null +++ b/.github/workflows/mtr/zenfs_parallel.cnf @@ -0,0 +1,2 @@ +[mysqld] +rocksdb-fs-uri=@envarray.FS_URI \ No newline at end of file diff --git a/.github/workflows/mtr/zenfs_rpl.cnf b/.github/workflows/mtr/zenfs_rpl.cnf new file mode 100644 index 00000000..ae963dd5 --- /dev/null +++ b/.github/workflows/mtr/zenfs_rpl.cnf @@ -0,0 +1,5 @@ +[mysqld] +rocksdb-fs-uri=@envarray.REPLICA_FS_URI + +[mysqld.1] +rocksdb-fs-uri=@envarray.SOURCE_FS_URI \ No newline at end of file diff --git a/.github/workflows/percona-server-smoke-test.yaml b/.github/workflows/percona-server-smoke-test.yaml index 868ebde7..b7c69080 100644 --- a/.github/workflows/percona-server-smoke-test.yaml +++ b/.github/workflows/percona-server-smoke-test.yaml @@ -11,26 +11,61 @@ on: name: Percona Server Smoke Test jobs: smoke-test: - runs-on: self-hosted + runs-on: nvme-zns steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Create context directory - run: cd harness && mkdir percona-context + + - name: Checkout zenfs for RocksDB + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: zenfs + + - name: Create build directory + run: | + mkdir -p percona-build-dir + + - name: Create result directory + run: | + mkdir -p percona-results + - name: Build docker image - run: cd harness && st -o logs/make-smoke -- make NO_VAGRANT=1 V=1 percona-debian-run + run: | + podman build --cgroup-manager cgroupfs \ + -t percona-debian \ + -f zenfs/.github/workflows/containerfiles/Dockerfile.percona-debian-build \ + --build-arg PERCONA_SERVER_BASE=Percona-Server-8.0.34-26 \ + --build-arg ZENFS_BRANCH=${GITHUB_SHA} \ + --build-arg ZENFS_REPO=https://github.com/${GITHUB_REPOSITORY}.git \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + - name: Run smoke test - run: cd harness && st --silent -o logs/make-smoke -- disk-select --disk 1TB -- make NO_VAGRANT=1 PERCONA_SERVER_BASE=Percona-Server-8.0.34-26 ZENFS_BRANCH=${GITHUB_SHA} ZENFS_REPO=https://github.com/${GITHUB_REPOSITORY}.git results/percona-server-smoke.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ${GITHUB_WORKSPACE}/percona-results:/percona/work/results \ + --volume ${GITHUB_WORKSPACE}/zenfs:/zenfs \ + percona-debian \ + bash /zenfs/.github/workflows/percona-server-smoke-test/zns_myrocks_zenfs_smoke.sh ${ZBD0} + - name: Archive unofficial experimental percona-server deb packages uses: actions/upload-artifact@v3 with: name: unofficial-experimental-zenfs-percona-server-deb-packages - path: harness/artifacts/unofficial-experimental-zenfs-percona-server-deb-packages.tgz + path: ./percona-results/unofficial-experimental-zenfs-percona-server-deb-packages.tgz + + - name: Remove build dir + run: rm -rf percona-build-dir + if: always() + + - name: Remove result dir + run: rm -rf percona-results + if: always() + - name: Remove images run: podman rmi --force --all if: always() diff --git a/.github/workflows/percona-server-smoke-test/zns_myrocks_zenfs_smoke.sh b/.github/workflows/percona-server-smoke-test/zns_myrocks_zenfs_smoke.sh new file mode 100755 index 00000000..da71b065 --- /dev/null +++ b/.github/workflows/percona-server-smoke-test/zns_myrocks_zenfs_smoke.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e +set -x + +DEV=$1 +if [ ! -b "/dev/$DEV" ]; then + echo "/dev/$DEV does not exist." + exit 1 +fi + +# Copy debian packages to the results dir so it can be collected later +cd /percona/work +tar czf /percona/work/results/unofficial-experimental-zenfs-percona-server-deb-packages.tgz *.deb +cd - + +# Createing the ZenFS device +chown mysql:mysql /dev/$DEV +sudo -H -u mysql zenfs mkfs --zbd=$DEV --aux_path=/tmp/mysql_zenfs_aux_$DEV --finish_threshold=0 --force + +# Restarting mysql with RocksDB - ZenFS enabled +service mysql stop + +sudo rm -r /var/lib/mysql +sudo mkdir /var/lib/mysql +chown -R mysql:mysql /var/lib/mysql +echo "loose-rocksdb-fs-uri=zenfs://dev:$DEV" >> /etc/mysql/mysql.conf.d/mysqld.cnf +echo "plugin-load-add=rocksdb=ha_rocksdb.so" >> /etc/mysql/mysql.conf.d/mysqld.cnf +echo "MySQL config:" +cat /etc/mysql/mysql.conf.d/mysqld.cnf + +mysqld --initialize-insecure --user mysql +service mysql restart +mysql -u root -e "SET GLOBAL default_storage_engine=ROCKSDB;" -S /var/run/mysqld/mysqld.sock +sleep 5 + +# Smoketest workload +mysql -u root -e "create database sbtest;" +mysql -u root -e "show engines;" +mysql -u root -e "show databases;" +sudo /percona/work/sysbench/src/lua/oltp_write_only.lua --db-driver=mysql --mysql-user=root --time=0 --create_secondary=off --mysql-host=localhost --mysql-db=sbtest --mysql-storage-engine=rocksdb --table-size=100000 --tables=4 --threads=4 --report-interval=5 prepare + +# Make sure that rocksdb's files end up on the ZNS device +sudo -H -u mysql zenfs list --path=.rocksdb --zbd=$DEV +sudo -H -u mysql zenfs list --path=.rocksdb --zbd=$DEV | grep IDENTITY diff --git a/.github/workflows/rocks-nightly.yaml b/.github/workflows/rocks-nightly.yaml index aea1aa58..3691c246 100644 --- a/.github/workflows/rocks-nightly.yaml +++ b/.github/workflows/rocks-nightly.yaml @@ -5,7 +5,7 @@ on: name: Smoke RocksDB Nightly jobs: smoke-nightly: - runs-on: self-hosted + runs-on: nvme-zns name: Smoke Nightly ${{ matrix.rocksdb-ref }} strategy: fail-fast: false @@ -19,34 +19,51 @@ jobs: steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: ${{ matrix.rocksdb-ref }} - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs - - name: Build docker image - run: cd harness && st -o logs/make-smoke -- make NO_VAGRANT=1 V=1 rocksdb-docker-image - - name: Run smoke tests - run: cd harness && st --silent -o logs/make-smoke -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zenfs-smoke.xml + path: rocksdb/plugin/zenfs + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + + - name: RocksDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" + + - name: Run smoke tests for RocksDB + run: | + podman run --cgroup-manager cgroupfs --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) smoke" + - name: Run utils tests - run: cd harness && st --silent -o logs/make-utils -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zenfs-utils.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "ZDEV=${ZBD0} FS_PARAMS=--fs_uri=zenfs://dev:${ZBD0} TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) utils" + - name: Remove docker images run: podman rmi --force --all if: always() diff --git a/.github/workflows/short-performance-xfs.yaml b/.github/workflows/short-performance-xfs.yaml index c39a9b45..64e1c614 100644 --- a/.github/workflows/short-performance-xfs.yaml +++ b/.github/workflows/short-performance-xfs.yaml @@ -5,35 +5,86 @@ name: Short Performance XFS Test jobs: short-performance-xfs: name: Short Performance XFS - runs-on: self-hosted + runs-on: nvme-zns timeout-minutes: 4320 steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs - - name: Run short performance test 1TB - run: cd harness && st --silent -o logs/make -- sudo with-xfs --disk 1TB -- make NO_VAGRANT=1 results/xfs-short.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload + path: rocksdb/plugin/zenfs + + - name: Create results directory + run: | + mkdir -p results + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Format conventional block device + run: | + sudo nvme format /dev/${BDEV0} -f --ses=1 + + - name: Create xfs on BDEV + run: | + sudo mkfs.xfs -f /dev/${BDEV0} + + - name: Create xfs mountpoint + run: | + mkdir -p xfs-short-performance + + - name: Mount BDEV to xfs mountpoint + run: | + sudo mount /dev/${BDEV0} xfs-short-performance + + - name: Change back ownership of mountpoint + run: | + sudo chown $(id -u):$(id -g) xfs-short-performance + + - name: Run short performance test + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ./results:/rocks/work/results:Z \ + --volume ./xfs-short-performance:/mnt/xfs:Z \ + --security-opt unmask=/sys/dev/block \ + rocksdb-zenfs \ + bash -c "FS_PARAMS='--db=/mnt/xfs --target_file_size_base=4080218931 --use_direct_io_for_flush_and_compaction --max_bytes_for_level_multiplier=4 --max_background_jobs=8 --use_direct_reads --write_buffer_size=2147483648' PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-xfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) quick_performance" + + - name: Archive short performance test results + uses: actions/upload-artifact@v3 + with: + name: quick_performance + path: ./results + if: always() + + - name: Remove result dir + run: rm -rf results if: always() + + - name: Umount xfs mountpoint + run: sudo umount -f xfs-short-performance + if: always() + + - name: Remove xfs mountpoint + run: rm -rf xfs-short-performance + if: always() + - name: Remove images run: podman rmi --force --all if: always() diff --git a/.github/workflows/short-performance.yaml b/.github/workflows/short-performance.yaml index f73cce3c..7b344c6a 100644 --- a/.github/workflows/short-performance.yaml +++ b/.github/workflows/short-performance.yaml @@ -6,35 +6,67 @@ on: name: Short Performance Test jobs: quick-performance: - runs-on: self-hosted + runs-on: nvme-zns timeout-minutes: 1440 steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs + path: rocksdb/plugin/zenfs + + - name: Create results directory + run: | + mkdir -p results + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + + - name: RocksDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" + - name: Run short performance test - run: cd harness && st --silent -o logs/make -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zenfs-quick.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ./results:/rocks/work/results:Z \ + --security-opt unmask=/sys/dev/block \ + rocksdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results DB_BENCH_EXTRA_PARAMS=$(/rocks/dest/zenfs/test/get_good_db_bench_params_for_zenfs.sh ${ZBD0}) /rocks/dest/zenfs/test/run.sh rocksdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) quick_performance" + + - name: Archive short performance test results + uses: actions/upload-artifact@v3 + with: + name: quick_performance + path: ./results + if: always() + + - name: Remove result dir + run: rm -rf results if: always() + - name: Remove images run: podman rmi --force --all if: always() diff --git a/.github/workflows/smoke-test-debug.yaml b/.github/workflows/smoke-test-debug.yaml index 791569a7..99e209c3 100644 --- a/.github/workflows/smoke-test-debug.yaml +++ b/.github/workflows/smoke-test-debug.yaml @@ -2,44 +2,62 @@ on: push: branches: - master + - main pull_request: branches: - '**' name: Quick Test (Debug) jobs: smoke-test-debug: - runs-on: self-hosted + runs-on: nvme-zns steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - name: Checkout zbdbench - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs - - name: Build docker image - run: cd harness && st -o logs/make-smoke -- make NO_VAGRANT=1 V=1 rocksdb-docker-image - - name: Run smoke tests - run: cd harness && st --silent -o logs/make-smoke -- disk-select --disk 1TB -- make NO_VAGRANT=1 ROCKSDB_DEBUG_LEVEL=1 results/zenfs-smoke.xml + path: rocksdb/plugin/zenfs + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + + - name: RocksDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" + + - name: Run smoke tests for RocksDB + run: | + podman run --cgroup-manager cgroupfs --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) smoke" + - name: Run utils tests - run: cd harness && st --silent -o logs/make-utils -- disk-select --disk 1TB -- make NO_VAGRANT=1 ROCKSDB_DEBUG_LEVEL=1 results/zenfs-utils.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "ZDEV=${ZBD0} FS_PARAMS=--fs_uri=zenfs://dev:${ZBD0} TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) utils" + - name: Remove images run: podman rmi --force --all if: always() diff --git a/.github/workflows/smoke-test.yaml b/.github/workflows/smoke-test.yaml index ac57e4fd..c3eb9bab 100644 --- a/.github/workflows/smoke-test.yaml +++ b/.github/workflows/smoke-test.yaml @@ -2,82 +2,98 @@ on: push: branches: - master + - main pull_request: branches: - '**' name: Quick Test jobs: smoke-test: - runs-on: self-hosted + runs-on: nvme-zns steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git - - - name: Checkout zbdbench for RocksDB - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/rocksdb-context/zbdbench - - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb - - - name: Checkout zbdbench for TerarkDB - uses: actions/checkout@v2 - with: - repository: westerndigitalcorporation/zbdbench - ref: v0.1.1 - path: harness/terarkdb-context/zbdbench + path: rocksdb - name: Checkout TerarkDB - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: bytedance/terarkdb ref: dev.1.4 - path: harness/terarkdb-context/terarkdb + path: terarkdb submodules: recursive - name: Checkout zenfs for RocksDB - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs + path: rocksdb/plugin/zenfs - name: Remove zenfs TerarkDB submodule - run: rm -rf harness/terarkdb-context/terarkdb/third-party/zenfs + run: rm -rf terarkdb/third-party/zenfs - name: Checkout zenfs for TerarkDB - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/terarkdb-context/terarkdb/third-party/zenfs + path: terarkdb/third-party/zenfs - name: Build RocksDB docker image - run: cd harness && st -o logs/make-rocksdb-image-$(uuidgen) -- make NO_VAGRANT=1 V=1 rocksdb-docker-image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . - - name: Build TerarkDB docker image - run: cd harness && st -o logs/make-terarkdb-image-$(uuidgen) -- make NO_VAGRANT=1 V=1 IMAGE=terarkdb terarkdb-docker-image + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler - - name: Run smoke tests for RocksDB - run: cd harness && st --silent -o logs/make-rocksdb-smoke-$(uuidgen) -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zenfs-smoke.xml + - name: RocksDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" - - name: Run smoke tests for TerarkDB - run: cd harness && st --silent -o logs/make-tetarkdb-smoke-$(uuidgen) -- disk-select --disk 1TB -- make NO_VAGRANT=1 IMAGE=terarkdb results/zenfs-smoke.xml + - name: Run smoke tests for RocksDB + run: | + podman run --cgroup-manager cgroupfs --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=rocksdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) smoke" - name: Run utils tests - run: cd harness && st --silent -o logs/make-utils -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zenfs-utils.xml + run: | + podman run --rm -i --device /dev/${ZBD0} \ + rocksdb-zenfs \ + bash -c "ZDEV=${ZBD0} FS_PARAMS=--fs_uri=zenfs://dev:${ZBD0} TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh rocksdb-zenfs$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) utils" - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload - if: always() + - name: Build TerarkDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t terarkdb-zenfs \ + -f terarkdb/third-party/zenfs/.github/workflows/containerfiles/Dockerfile.terarkdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + . + + - name: TerarkDB mkfs zenfs + run: | + podman run --rm -i --device /dev/${ZBD0} \ + terarkdb-zenfs \ + bash -c "zenfs mkfs --zbd ${ZBD0} --aux_path /tmp/zenfs-aux --force" + + - name: Run smoke tests for TerarkDB + run: | + podman run --rm -i --device /dev/${ZBD0} \ + terarkdb-zenfs \ + bash -c "FS_PARAMS=--fs-uri=zenfs://dev:${ZBD0} PLUGIN_HOST=terarkdb TOOLS_DIR=/rocks/dest/bin ZENFS_DIR=/rocks/dest/bin OUTPUT_DIR=/rocks/work/results /rocks/dest/zenfs/test/run.sh terarkdb-zenfs-$(date '+%Y-%m-%d_%H-%M-%S' --utc)-$(uuidgen) smoke" - name: Remove images run: podman rmi --force --all diff --git a/.github/workflows/zbdbench.yaml b/.github/workflows/zbdbench.yaml index e1c9b2de..6910255e 100644 --- a/.github/workflows/zbdbench.yaml +++ b/.github/workflows/zbdbench.yaml @@ -6,35 +6,62 @@ on: name: ZBDBench jobs: zbdbench: - runs-on: self-hosted + runs-on: nvme-zns timeout-minutes: 1440 steps: - name: Clean run: rm -rf ${GITHUB_WORKSPACE}/* - - name: Checkout build scripts - run: git clone ~/harness.git + - name: Checkout zbdbench - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: westerndigitalcorporation/zbdbench ref: v0.1.1 - path: harness/rocksdb-context/zbdbench + path: zbdbench + - name: Checkout rocksdb - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: facebook/rocksdb ref: v7.0.2 - path: harness/rocksdb-context/rocksdb + path: rocksdb + - name: Checkout zenfs - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - path: harness/rocksdb-context/rocksdb/plugin/zenfs + path: rocksdb/plugin/zenfs + + - name: Build RocksDB docker image + run: | + podman build --cgroup-manager cgroupfs \ + -t rocksdb-zenfs \ + -f rocksdb/plugin/zenfs/.github/workflows/containerfiles/Dockerfile.rocksdb-zenfs \ + --build-arg LIBZBD_GIT_CHECKOUT=v2.0.2 \ + --build-arg ROCKSDB_DEBUG_LEVEL=0 \ + . + + - name: Set mq-deadline scheduler for the ZBD + run: | + echo deadline > /sys/block/${ZBD0}/queue/scheduler + - name: Run ZBDBench - run: cd harness && st --silent -o logs/make -- disk-select --disk 1TB -- make NO_VAGRANT=1 results/zbdbench.xml - - name: Collect Results - run: cd harness && make NO_VAGRANT=1 upload + run: | + podman run --rm -i --device /dev/${ZBD0} \ + --volume ./zbdbench:/rocks/dest/zbdbench:Z \ + --security-opt unmask=/sys/dev/block \ + rocksdb-zenfs \ + python3 /rocks/dest/zbdbench/run.py -c system \ + -b rocksdb_fillprep rocksdb_overwrite rocksdb_readwhilewriting \ + -d /dev/${ZBD0} + + - name: Archive ZBDBench test results + uses: actions/upload-artifact@v3 + with: + name: zbdbench-rocksdb + path: ./zbdbench/zbdbench_results if: always() + - name: Remove images run: podman rmi --force --all if: always() diff --git a/fs/io_zenfs.h b/fs/io_zenfs.h index 022a4aea..5ebd7d61 100644 --- a/fs/io_zenfs.h +++ b/fs/io_zenfs.h @@ -226,6 +226,11 @@ class ZonedWritableFile : public FSWritableFile { virtual IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) { + (void)options; + (void)dbg; + return zoneFile_->GetFileSize(); + } bool use_direct_io() const override { return !buffered; } bool IsSyncThreadSafe() const override { return true; }; size_t GetRequiredBufferAlignment() const override {